agora inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH v17 7/8] Row pattern recognition patch (tests).
156+ messages / 2 participants
[nested] [flat]

* [PATCH v17 7/8] Row pattern recognition patch (tests).
@ 2024-04-28 11:00 Tatsuo Ishii <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Tatsuo Ishii @ 2024-04-28 11:00 UTC (permalink / raw)

---
 src/test/regress/expected/rpr.out  | 821 +++++++++++++++++++++++++++++
 src/test/regress/parallel_schedule |   2 +-
 src/test/regress/sql/rpr.sql       | 392 ++++++++++++++
 3 files changed, 1214 insertions(+), 1 deletion(-)
 create mode 100644 src/test/regress/expected/rpr.out
 create mode 100644 src/test/regress/sql/rpr.sql

diff --git a/src/test/regress/expected/rpr.out b/src/test/regress/expected/rpr.out
new file mode 100644
index 0000000000..5a0bffa9f2
--- /dev/null
+++ b/src/test/regress/expected/rpr.out
@@ -0,0 +1,821 @@
+--
+-- Test for row pattern definition clause
+--
+CREATE TEMP TABLE stock (
+       company TEXT,
+       tdate DATE,
+       price INTEGER
+);
+INSERT INTO stock VALUES ('company1', '2023-07-01', 100);
+INSERT INTO stock VALUES ('company1', '2023-07-02', 200);
+INSERT INTO stock VALUES ('company1', '2023-07-03', 150);
+INSERT INTO stock VALUES ('company1', '2023-07-04', 140);
+INSERT INTO stock VALUES ('company1', '2023-07-05', 150);
+INSERT INTO stock VALUES ('company1', '2023-07-06', 90);
+INSERT INTO stock VALUES ('company1', '2023-07-07', 110);
+INSERT INTO stock VALUES ('company1', '2023-07-08', 130);
+INSERT INTO stock VALUES ('company1', '2023-07-09', 120);
+INSERT INTO stock VALUES ('company1', '2023-07-10', 130);
+INSERT INTO stock VALUES ('company2', '2023-07-01', 50);
+INSERT INTO stock VALUES ('company2', '2023-07-02', 2000);
+INSERT INTO stock VALUES ('company2', '2023-07-03', 1500);
+INSERT INTO stock VALUES ('company2', '2023-07-04', 1400);
+INSERT INTO stock VALUES ('company2', '2023-07-05', 1500);
+INSERT INTO stock VALUES ('company2', '2023-07-06', 60);
+INSERT INTO stock VALUES ('company2', '2023-07-07', 1100);
+INSERT INTO stock VALUES ('company2', '2023-07-08', 1300);
+INSERT INTO stock VALUES ('company2', '2023-07-09', 1200);
+INSERT INTO stock VALUES ('company2', '2023-07-10', 1300);
+SELECT * FROM stock;
+ company  |   tdate    | price 
+----------+------------+-------
+ company1 | 07-01-2023 |   100
+ company1 | 07-02-2023 |   200
+ company1 | 07-03-2023 |   150
+ company1 | 07-04-2023 |   140
+ company1 | 07-05-2023 |   150
+ company1 | 07-06-2023 |    90
+ company1 | 07-07-2023 |   110
+ company1 | 07-08-2023 |   130
+ company1 | 07-09-2023 |   120
+ company1 | 07-10-2023 |   130
+ company2 | 07-01-2023 |    50
+ company2 | 07-02-2023 |  2000
+ company2 | 07-03-2023 |  1500
+ company2 | 07-04-2023 |  1400
+ company2 | 07-05-2023 |  1500
+ company2 | 07-06-2023 |    60
+ company2 | 07-07-2023 |  1100
+ company2 | 07-08-2023 |  1300
+ company2 | 07-09-2023 |  1200
+ company2 | 07-10-2023 |  1300
+(20 rows)
+
+-- basic test using PREV
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w,
+ nth_value(tdate, 2) OVER w AS nth_second
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+ company  |   tdate    | price | first_value | last_value | nth_second 
+----------+------------+-------+-------------+------------+------------
+ company1 | 07-01-2023 |   100 |         100 |        140 | 07-02-2023
+ company1 | 07-02-2023 |   200 |             |            | 
+ company1 | 07-03-2023 |   150 |             |            | 
+ company1 | 07-04-2023 |   140 |             |            | 
+ company1 | 07-05-2023 |   150 |             |            | 
+ company1 | 07-06-2023 |    90 |          90 |        120 | 07-07-2023
+ company1 | 07-07-2023 |   110 |             |            | 
+ company1 | 07-08-2023 |   130 |             |            | 
+ company1 | 07-09-2023 |   120 |             |            | 
+ company1 | 07-10-2023 |   130 |             |            | 
+ company2 | 07-01-2023 |    50 |          50 |       1400 | 07-02-2023
+ company2 | 07-02-2023 |  2000 |             |            | 
+ company2 | 07-03-2023 |  1500 |             |            | 
+ company2 | 07-04-2023 |  1400 |             |            | 
+ company2 | 07-05-2023 |  1500 |             |            | 
+ company2 | 07-06-2023 |    60 |          60 |       1200 | 07-07-2023
+ company2 | 07-07-2023 |  1100 |             |            | 
+ company2 | 07-08-2023 |  1300 |             |            | 
+ company2 | 07-09-2023 |  1200 |             |            | 
+ company2 | 07-10-2023 |  1300 |             |            | 
+(20 rows)
+
+-- basic test using PREV. UP appears twice
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w,
+ nth_value(tdate, 2) OVER w AS nth_second
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+ UP+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+ company  |   tdate    | price | first_value | last_value | nth_second 
+----------+------------+-------+-------------+------------+------------
+ company1 | 07-01-2023 |   100 |         100 |        150 | 07-02-2023
+ company1 | 07-02-2023 |   200 |             |            | 
+ company1 | 07-03-2023 |   150 |             |            | 
+ company1 | 07-04-2023 |   140 |             |            | 
+ company1 | 07-05-2023 |   150 |             |            | 
+ company1 | 07-06-2023 |    90 |          90 |        130 | 07-07-2023
+ company1 | 07-07-2023 |   110 |             |            | 
+ company1 | 07-08-2023 |   130 |             |            | 
+ company1 | 07-09-2023 |   120 |             |            | 
+ company1 | 07-10-2023 |   130 |             |            | 
+ company2 | 07-01-2023 |    50 |          50 |       1500 | 07-02-2023
+ company2 | 07-02-2023 |  2000 |             |            | 
+ company2 | 07-03-2023 |  1500 |             |            | 
+ company2 | 07-04-2023 |  1400 |             |            | 
+ company2 | 07-05-2023 |  1500 |             |            | 
+ company2 | 07-06-2023 |    60 |          60 |       1300 | 07-07-2023
+ company2 | 07-07-2023 |  1100 |             |            | 
+ company2 | 07-08-2023 |  1300 |             |            | 
+ company2 | 07-09-2023 |  1200 |             |            | 
+ company2 | 07-10-2023 |  1300 |             |            | 
+(20 rows)
+
+-- basic test using PREV. Use '*'
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w,
+ nth_value(tdate, 2) OVER w AS nth_second
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP* DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+ company  |   tdate    | price | first_value | last_value | nth_second 
+----------+------------+-------+-------------+------------+------------
+ company1 | 07-01-2023 |   100 |         100 |        140 | 07-02-2023
+ company1 | 07-02-2023 |   200 |             |            | 
+ company1 | 07-03-2023 |   150 |             |            | 
+ company1 | 07-04-2023 |   140 |             |            | 
+ company1 | 07-05-2023 |   150 |         150 |         90 | 07-06-2023
+ company1 | 07-06-2023 |    90 |             |            | 
+ company1 | 07-07-2023 |   110 |         110 |        120 | 07-08-2023
+ company1 | 07-08-2023 |   130 |             |            | 
+ company1 | 07-09-2023 |   120 |             |            | 
+ company1 | 07-10-2023 |   130 |             |            | 
+ company2 | 07-01-2023 |    50 |          50 |       1400 | 07-02-2023
+ company2 | 07-02-2023 |  2000 |             |            | 
+ company2 | 07-03-2023 |  1500 |             |            | 
+ company2 | 07-04-2023 |  1400 |             |            | 
+ company2 | 07-05-2023 |  1500 |        1500 |         60 | 07-06-2023
+ company2 | 07-06-2023 |    60 |             |            | 
+ company2 | 07-07-2023 |  1100 |        1100 |       1200 | 07-08-2023
+ company2 | 07-08-2023 |  1300 |             |            | 
+ company2 | 07-09-2023 |  1200 |             |            | 
+ company2 | 07-10-2023 |  1300 |             |            | 
+(20 rows)
+
+-- basic test with none greedy pattern
+SELECT company, tdate, price, count(*) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (A A A)
+ DEFINE
+  A AS price >= 140 AND price <= 150
+);
+ company  |   tdate    | price | count 
+----------+------------+-------+-------
+ company1 | 07-01-2023 |   100 |     0
+ company1 | 07-02-2023 |   200 |     0
+ company1 | 07-03-2023 |   150 |     3
+ company1 | 07-04-2023 |   140 |     0
+ company1 | 07-05-2023 |   150 |     0
+ company1 | 07-06-2023 |    90 |     0
+ company1 | 07-07-2023 |   110 |     0
+ company1 | 07-08-2023 |   130 |     0
+ company1 | 07-09-2023 |   120 |     0
+ company1 | 07-10-2023 |   130 |     0
+ company2 | 07-01-2023 |    50 |     0
+ company2 | 07-02-2023 |  2000 |     0
+ company2 | 07-03-2023 |  1500 |     0
+ company2 | 07-04-2023 |  1400 |     0
+ company2 | 07-05-2023 |  1500 |     0
+ company2 | 07-06-2023 |    60 |     0
+ company2 | 07-07-2023 |  1100 |     0
+ company2 | 07-08-2023 |  1300 |     0
+ company2 | 07-09-2023 |  1200 |     0
+ company2 | 07-10-2023 |  1300 |     0
+(20 rows)
+
+-- last_value() should remain consistent
+SELECT company, tdate, price, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+ company  |   tdate    | price | last_value 
+----------+------------+-------+------------
+ company1 | 07-01-2023 |   100 |        140
+ company1 | 07-02-2023 |   200 |           
+ company1 | 07-03-2023 |   150 |           
+ company1 | 07-04-2023 |   140 |           
+ company1 | 07-05-2023 |   150 |           
+ company1 | 07-06-2023 |    90 |        120
+ company1 | 07-07-2023 |   110 |           
+ company1 | 07-08-2023 |   130 |           
+ company1 | 07-09-2023 |   120 |           
+ company1 | 07-10-2023 |   130 |           
+ company2 | 07-01-2023 |    50 |       1400
+ company2 | 07-02-2023 |  2000 |           
+ company2 | 07-03-2023 |  1500 |           
+ company2 | 07-04-2023 |  1400 |           
+ company2 | 07-05-2023 |  1500 |           
+ company2 | 07-06-2023 |    60 |       1200
+ company2 | 07-07-2023 |  1100 |           
+ company2 | 07-08-2023 |  1300 |           
+ company2 | 07-09-2023 |  1200 |           
+ company2 | 07-10-2023 |  1300 |           
+(20 rows)
+
+-- omit "START" in DEFINE but it is ok because "START AS TRUE" is
+-- implicitly defined. per spec.
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w,
+ nth_value(tdate, 2) OVER w AS nth_second
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+ company  |   tdate    | price | first_value | last_value | nth_second 
+----------+------------+-------+-------------+------------+------------
+ company1 | 07-01-2023 |   100 |         100 |        140 | 07-02-2023
+ company1 | 07-02-2023 |   200 |             |            | 
+ company1 | 07-03-2023 |   150 |             |            | 
+ company1 | 07-04-2023 |   140 |             |            | 
+ company1 | 07-05-2023 |   150 |             |            | 
+ company1 | 07-06-2023 |    90 |          90 |        120 | 07-07-2023
+ company1 | 07-07-2023 |   110 |             |            | 
+ company1 | 07-08-2023 |   130 |             |            | 
+ company1 | 07-09-2023 |   120 |             |            | 
+ company1 | 07-10-2023 |   130 |             |            | 
+ company2 | 07-01-2023 |    50 |          50 |       1400 | 07-02-2023
+ company2 | 07-02-2023 |  2000 |             |            | 
+ company2 | 07-03-2023 |  1500 |             |            | 
+ company2 | 07-04-2023 |  1400 |             |            | 
+ company2 | 07-05-2023 |  1500 |             |            | 
+ company2 | 07-06-2023 |    60 |          60 |       1200 | 07-07-2023
+ company2 | 07-07-2023 |  1100 |             |            | 
+ company2 | 07-08-2023 |  1300 |             |            | 
+ company2 | 07-09-2023 |  1200 |             |            | 
+ company2 | 07-10-2023 |  1300 |             |            | 
+(20 rows)
+
+-- the first row start with less than or equal to 100
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (LOWPRICE UP+ DOWN+)
+ DEFINE
+  LOWPRICE AS price <= 100,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+ company  |   tdate    | price | first_value | last_value 
+----------+------------+-------+-------------+------------
+ company1 | 07-01-2023 |   100 |         100 |        140
+ company1 | 07-02-2023 |   200 |             |           
+ company1 | 07-03-2023 |   150 |             |           
+ company1 | 07-04-2023 |   140 |             |           
+ company1 | 07-05-2023 |   150 |             |           
+ company1 | 07-06-2023 |    90 |          90 |        120
+ company1 | 07-07-2023 |   110 |             |           
+ company1 | 07-08-2023 |   130 |             |           
+ company1 | 07-09-2023 |   120 |             |           
+ company1 | 07-10-2023 |   130 |             |           
+ company2 | 07-01-2023 |    50 |          50 |       1400
+ company2 | 07-02-2023 |  2000 |             |           
+ company2 | 07-03-2023 |  1500 |             |           
+ company2 | 07-04-2023 |  1400 |             |           
+ company2 | 07-05-2023 |  1500 |             |           
+ company2 | 07-06-2023 |    60 |          60 |       1200
+ company2 | 07-07-2023 |  1100 |             |           
+ company2 | 07-08-2023 |  1300 |             |           
+ company2 | 07-09-2023 |  1200 |             |           
+ company2 | 07-10-2023 |  1300 |             |           
+(20 rows)
+
+-- second row raises 120%
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (LOWPRICE UP+ DOWN+)
+ DEFINE
+  LOWPRICE AS price <= 100,
+  UP AS price > PREV(price) * 1.2,
+  DOWN AS price < PREV(price)
+);
+ company  |   tdate    | price | first_value | last_value 
+----------+------------+-------+-------------+------------
+ company1 | 07-01-2023 |   100 |         100 |        140
+ company1 | 07-02-2023 |   200 |             |           
+ company1 | 07-03-2023 |   150 |             |           
+ company1 | 07-04-2023 |   140 |             |           
+ company1 | 07-05-2023 |   150 |             |           
+ company1 | 07-06-2023 |    90 |             |           
+ company1 | 07-07-2023 |   110 |             |           
+ company1 | 07-08-2023 |   130 |             |           
+ company1 | 07-09-2023 |   120 |             |           
+ company1 | 07-10-2023 |   130 |             |           
+ company2 | 07-01-2023 |    50 |          50 |       1400
+ company2 | 07-02-2023 |  2000 |             |           
+ company2 | 07-03-2023 |  1500 |             |           
+ company2 | 07-04-2023 |  1400 |             |           
+ company2 | 07-05-2023 |  1500 |             |           
+ company2 | 07-06-2023 |    60 |             |           
+ company2 | 07-07-2023 |  1100 |             |           
+ company2 | 07-08-2023 |  1300 |             |           
+ company2 | 07-09-2023 |  1200 |             |           
+ company2 | 07-10-2023 |  1300 |             |           
+(20 rows)
+
+-- using NEXT
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UPDOWN)
+ DEFINE
+  START AS TRUE,
+  UPDOWN AS price > PREV(price) AND price > NEXT(price)
+);
+ company  |   tdate    | price | first_value | last_value 
+----------+------------+-------+-------------+------------
+ company1 | 07-01-2023 |   100 |         100 |        200
+ company1 | 07-02-2023 |   200 |             |           
+ company1 | 07-03-2023 |   150 |             |           
+ company1 | 07-04-2023 |   140 |         140 |        150
+ company1 | 07-05-2023 |   150 |             |           
+ company1 | 07-06-2023 |    90 |             |           
+ company1 | 07-07-2023 |   110 |         110 |        130
+ company1 | 07-08-2023 |   130 |             |           
+ company1 | 07-09-2023 |   120 |             |           
+ company1 | 07-10-2023 |   130 |             |           
+ company2 | 07-01-2023 |    50 |          50 |       2000
+ company2 | 07-02-2023 |  2000 |             |           
+ company2 | 07-03-2023 |  1500 |             |           
+ company2 | 07-04-2023 |  1400 |        1400 |       1500
+ company2 | 07-05-2023 |  1500 |             |           
+ company2 | 07-06-2023 |    60 |             |           
+ company2 | 07-07-2023 |  1100 |        1100 |       1300
+ company2 | 07-08-2023 |  1300 |             |           
+ company2 | 07-09-2023 |  1200 |             |           
+ company2 | 07-10-2023 |  1300 |             |           
+(20 rows)
+
+-- using AFTER MATCH SKIP TO NEXT ROW
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ AFTER MATCH SKIP TO NEXT ROW
+ INITIAL
+ PATTERN (START UPDOWN)
+ DEFINE
+  START AS TRUE,
+  UPDOWN AS price > PREV(price) AND price > NEXT(price)
+);
+ company  |   tdate    | price | first_value | last_value 
+----------+------------+-------+-------------+------------
+ company1 | 07-01-2023 |   100 |         100 |        200
+ company1 | 07-02-2023 |   200 |             |           
+ company1 | 07-03-2023 |   150 |             |           
+ company1 | 07-04-2023 |   140 |         140 |        150
+ company1 | 07-05-2023 |   150 |             |           
+ company1 | 07-06-2023 |    90 |             |           
+ company1 | 07-07-2023 |   110 |         110 |        130
+ company1 | 07-08-2023 |   130 |             |           
+ company1 | 07-09-2023 |   120 |             |           
+ company1 | 07-10-2023 |   130 |             |           
+ company2 | 07-01-2023 |    50 |          50 |       2000
+ company2 | 07-02-2023 |  2000 |             |           
+ company2 | 07-03-2023 |  1500 |             |           
+ company2 | 07-04-2023 |  1400 |        1400 |       1500
+ company2 | 07-05-2023 |  1500 |             |           
+ company2 | 07-06-2023 |    60 |             |           
+ company2 | 07-07-2023 |  1100 |        1100 |       1300
+ company2 | 07-08-2023 |  1300 |             |           
+ company2 | 07-09-2023 |  1200 |             |           
+ company2 | 07-10-2023 |  1300 |             |           
+(20 rows)
+
+-- match everything
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ AFTER MATCH SKIP PAST LAST ROW
+ INITIAL
+ PATTERN (A+)
+ DEFINE
+  A AS TRUE
+);
+ company  |   tdate    | price | first_value | last_value 
+----------+------------+-------+-------------+------------
+ company1 | 07-01-2023 |   100 |         100 |        130
+ company1 | 07-02-2023 |   200 |             |           
+ company1 | 07-03-2023 |   150 |             |           
+ company1 | 07-04-2023 |   140 |             |           
+ company1 | 07-05-2023 |   150 |             |           
+ company1 | 07-06-2023 |    90 |             |           
+ company1 | 07-07-2023 |   110 |             |           
+ company1 | 07-08-2023 |   130 |             |           
+ company1 | 07-09-2023 |   120 |             |           
+ company1 | 07-10-2023 |   130 |             |           
+ company2 | 07-01-2023 |    50 |          50 |       1300
+ company2 | 07-02-2023 |  2000 |             |           
+ company2 | 07-03-2023 |  1500 |             |           
+ company2 | 07-04-2023 |  1400 |             |           
+ company2 | 07-05-2023 |  1500 |             |           
+ company2 | 07-06-2023 |    60 |             |           
+ company2 | 07-07-2023 |  1100 |             |           
+ company2 | 07-08-2023 |  1300 |             |           
+ company2 | 07-09-2023 |  1200 |             |           
+ company2 | 07-10-2023 |  1300 |             |           
+(20 rows)
+
+-- backtracking with reclassification of rows
+-- using AFTER MATCH SKIP PAST LAST ROW
+SELECT company, tdate, price, first_value(tdate) OVER w, last_value(tdate) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ AFTER MATCH SKIP PAST LAST ROW
+ INITIAL
+ PATTERN (A+ B+)
+ DEFINE
+  A AS price > 100,
+  B AS price > 100
+);
+ company  |   tdate    | price | first_value | last_value 
+----------+------------+-------+-------------+------------
+ company1 | 07-01-2023 |   100 |             | 
+ company1 | 07-02-2023 |   200 | 07-02-2023  | 07-05-2023
+ company1 | 07-03-2023 |   150 |             | 
+ company1 | 07-04-2023 |   140 |             | 
+ company1 | 07-05-2023 |   150 |             | 
+ company1 | 07-06-2023 |    90 |             | 
+ company1 | 07-07-2023 |   110 | 07-07-2023  | 07-10-2023
+ company1 | 07-08-2023 |   130 |             | 
+ company1 | 07-09-2023 |   120 |             | 
+ company1 | 07-10-2023 |   130 |             | 
+ company2 | 07-01-2023 |    50 |             | 
+ company2 | 07-02-2023 |  2000 | 07-02-2023  | 07-05-2023
+ company2 | 07-03-2023 |  1500 |             | 
+ company2 | 07-04-2023 |  1400 |             | 
+ company2 | 07-05-2023 |  1500 |             | 
+ company2 | 07-06-2023 |    60 |             | 
+ company2 | 07-07-2023 |  1100 | 07-07-2023  | 07-10-2023
+ company2 | 07-08-2023 |  1300 |             | 
+ company2 | 07-09-2023 |  1200 |             | 
+ company2 | 07-10-2023 |  1300 |             | 
+(20 rows)
+
+-- backtracking with reclassification of rows
+-- using AFTER MATCH SKIP TO NEXT ROW
+SELECT company, tdate, price, first_value(tdate) OVER w, last_value(tdate) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ AFTER MATCH SKIP TO NEXT ROW
+ INITIAL
+ PATTERN (A+ B+)
+ DEFINE
+  A AS price > 100,
+  B AS price > 100
+);
+ company  |   tdate    | price | first_value | last_value 
+----------+------------+-------+-------------+------------
+ company1 | 07-01-2023 |   100 |             | 
+ company1 | 07-02-2023 |   200 | 07-02-2023  | 07-05-2023
+ company1 | 07-03-2023 |   150 | 07-03-2023  | 07-05-2023
+ company1 | 07-04-2023 |   140 | 07-04-2023  | 07-05-2023
+ company1 | 07-05-2023 |   150 |             | 
+ company1 | 07-06-2023 |    90 |             | 
+ company1 | 07-07-2023 |   110 | 07-07-2023  | 07-10-2023
+ company1 | 07-08-2023 |   130 | 07-08-2023  | 07-10-2023
+ company1 | 07-09-2023 |   120 | 07-09-2023  | 07-10-2023
+ company1 | 07-10-2023 |   130 |             | 
+ company2 | 07-01-2023 |    50 |             | 
+ company2 | 07-02-2023 |  2000 | 07-02-2023  | 07-05-2023
+ company2 | 07-03-2023 |  1500 | 07-03-2023  | 07-05-2023
+ company2 | 07-04-2023 |  1400 | 07-04-2023  | 07-05-2023
+ company2 | 07-05-2023 |  1500 |             | 
+ company2 | 07-06-2023 |    60 |             | 
+ company2 | 07-07-2023 |  1100 | 07-07-2023  | 07-10-2023
+ company2 | 07-08-2023 |  1300 | 07-08-2023  | 07-10-2023
+ company2 | 07-09-2023 |  1200 | 07-09-2023  | 07-10-2023
+ company2 | 07-10-2023 |  1300 |             | 
+(20 rows)
+
+-- ROWS BETWEEN CURRENT ROW AND offset FOLLOWING
+SELECT company, tdate, price, first_value(tdate) OVER w, last_value(tdate) OVER w,
+ count(*) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND 2 FOLLOWING
+ AFTER MATCH SKIP PAST LAST ROW
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+ company  |   tdate    | price | first_value | last_value | count 
+----------+------------+-------+-------------+------------+-------
+ company1 | 07-01-2023 |   100 | 07-01-2023  | 07-03-2023 |     3
+ company1 | 07-02-2023 |   200 |             |            |     0
+ company1 | 07-03-2023 |   150 |             |            |     0
+ company1 | 07-04-2023 |   140 | 07-04-2023  | 07-06-2023 |     3
+ company1 | 07-05-2023 |   150 |             |            |     0
+ company1 | 07-06-2023 |    90 |             |            |     0
+ company1 | 07-07-2023 |   110 | 07-07-2023  | 07-09-2023 |     3
+ company1 | 07-08-2023 |   130 |             |            |     0
+ company1 | 07-09-2023 |   120 |             |            |     0
+ company1 | 07-10-2023 |   130 |             |            |     0
+ company2 | 07-01-2023 |    50 | 07-01-2023  | 07-03-2023 |     3
+ company2 | 07-02-2023 |  2000 |             |            |     0
+ company2 | 07-03-2023 |  1500 |             |            |     0
+ company2 | 07-04-2023 |  1400 | 07-04-2023  | 07-06-2023 |     3
+ company2 | 07-05-2023 |  1500 |             |            |     0
+ company2 | 07-06-2023 |    60 |             |            |     0
+ company2 | 07-07-2023 |  1100 | 07-07-2023  | 07-09-2023 |     3
+ company2 | 07-08-2023 |  1300 |             |            |     0
+ company2 | 07-09-2023 |  1200 |             |            |     0
+ company2 | 07-10-2023 |  1300 |             |            |     0
+(20 rows)
+
+--
+-- Aggregates
+--
+-- using AFTER MATCH SKIP PAST LAST ROW
+SELECT company, tdate, price,
+ first_value(price) OVER w,
+ last_value(price) OVER w,
+ max(price) OVER w,
+ min(price) OVER w,
+ sum(price) OVER w,
+ avg(price) OVER w,
+ count(price) OVER w
+FROM stock
+WINDOW w AS (
+PARTITION BY company
+ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+AFTER MATCH SKIP PAST LAST ROW
+INITIAL
+PATTERN (START UP+ DOWN+)
+DEFINE
+START AS TRUE,
+UP AS price > PREV(price),
+DOWN AS price < PREV(price)
+);
+ company  |   tdate    | price | first_value | last_value | max  | min | sum  |          avg          | count 
+----------+------------+-------+-------------+------------+------+-----+------+-----------------------+-------
+ company1 | 07-01-2023 |   100 |         100 |        140 |  200 | 100 |  590 |  147.5000000000000000 |     4
+ company1 | 07-02-2023 |   200 |             |            |      |     |      |                       |     0
+ company1 | 07-03-2023 |   150 |             |            |      |     |      |                       |     0
+ company1 | 07-04-2023 |   140 |             |            |      |     |      |                       |     0
+ company1 | 07-05-2023 |   150 |             |            |      |     |      |                       |     0
+ company1 | 07-06-2023 |    90 |          90 |        120 |  130 |  90 |  450 |  112.5000000000000000 |     4
+ company1 | 07-07-2023 |   110 |             |            |      |     |      |                       |     0
+ company1 | 07-08-2023 |   130 |             |            |      |     |      |                       |     0
+ company1 | 07-09-2023 |   120 |             |            |      |     |      |                       |     0
+ company1 | 07-10-2023 |   130 |             |            |      |     |      |                       |     0
+ company2 | 07-01-2023 |    50 |          50 |       1400 | 2000 |  50 | 4950 | 1237.5000000000000000 |     4
+ company2 | 07-02-2023 |  2000 |             |            |      |     |      |                       |     0
+ company2 | 07-03-2023 |  1500 |             |            |      |     |      |                       |     0
+ company2 | 07-04-2023 |  1400 |             |            |      |     |      |                       |     0
+ company2 | 07-05-2023 |  1500 |             |            |      |     |      |                       |     0
+ company2 | 07-06-2023 |    60 |          60 |       1200 | 1300 |  60 | 3660 |  915.0000000000000000 |     4
+ company2 | 07-07-2023 |  1100 |             |            |      |     |      |                       |     0
+ company2 | 07-08-2023 |  1300 |             |            |      |     |      |                       |     0
+ company2 | 07-09-2023 |  1200 |             |            |      |     |      |                       |     0
+ company2 | 07-10-2023 |  1300 |             |            |      |     |      |                       |     0
+(20 rows)
+
+-- using AFTER MATCH SKIP TO NEXT ROW
+SELECT company, tdate, price,
+ first_value(price) OVER w,
+ last_value(price) OVER w,
+ max(price) OVER w,
+ min(price) OVER w,
+ sum(price) OVER w,
+ avg(price) OVER w,
+ count(price) OVER w
+FROM stock
+WINDOW w AS (
+PARTITION BY company
+ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+AFTER MATCH SKIP TO NEXT ROW
+INITIAL
+PATTERN (START UP+ DOWN+)
+DEFINE
+START AS TRUE,
+UP AS price > PREV(price),
+DOWN AS price < PREV(price)
+);
+ company  |   tdate    | price | first_value | last_value | max  | min  | sum  |          avg          | count 
+----------+------------+-------+-------------+------------+------+------+------+-----------------------+-------
+ company1 | 07-01-2023 |   100 |         100 |        140 |  200 |  100 |  590 |  147.5000000000000000 |     4
+ company1 | 07-02-2023 |   200 |             |            |      |      |      |                       |     0
+ company1 | 07-03-2023 |   150 |             |            |      |      |      |                       |     0
+ company1 | 07-04-2023 |   140 |         140 |         90 |  150 |   90 |  380 |  126.6666666666666667 |     3
+ company1 | 07-05-2023 |   150 |             |            |      |      |      |                       |     0
+ company1 | 07-06-2023 |    90 |          90 |        120 |  130 |   90 |  450 |  112.5000000000000000 |     4
+ company1 | 07-07-2023 |   110 |         110 |        120 |  130 |  110 |  360 |  120.0000000000000000 |     3
+ company1 | 07-08-2023 |   130 |             |            |      |      |      |                       |     0
+ company1 | 07-09-2023 |   120 |             |            |      |      |      |                       |     0
+ company1 | 07-10-2023 |   130 |             |            |      |      |      |                       |     0
+ company2 | 07-01-2023 |    50 |          50 |       1400 | 2000 |   50 | 4950 | 1237.5000000000000000 |     4
+ company2 | 07-02-2023 |  2000 |             |            |      |      |      |                       |     0
+ company2 | 07-03-2023 |  1500 |             |            |      |      |      |                       |     0
+ company2 | 07-04-2023 |  1400 |        1400 |         60 | 1500 |   60 | 2960 |  986.6666666666666667 |     3
+ company2 | 07-05-2023 |  1500 |             |            |      |      |      |                       |     0
+ company2 | 07-06-2023 |    60 |          60 |       1200 | 1300 |   60 | 3660 |  915.0000000000000000 |     4
+ company2 | 07-07-2023 |  1100 |        1100 |       1200 | 1300 | 1100 | 3600 | 1200.0000000000000000 |     3
+ company2 | 07-08-2023 |  1300 |             |            |      |      |      |                       |     0
+ company2 | 07-09-2023 |  1200 |             |            |      |      |      |                       |     0
+ company2 | 07-10-2023 |  1300 |             |            |      |      |      |                       |     0
+(20 rows)
+
+-- JOIN case
+CREATE TEMP TABLE t1 (i int, v1 int);
+CREATE TEMP TABLE t2 (j int, v2 int);
+INSERT INTO t1 VALUES(1,10);
+INSERT INTO t1 VALUES(1,11);
+INSERT INTO t1 VALUES(1,12);
+INSERT INTO t2 VALUES(2,10);
+INSERT INTO t2 VALUES(2,11);
+INSERT INTO t2 VALUES(2,12);
+SELECT * FROM t1, t2 WHERE t1.v1 <= 11 AND t2.v2 <= 11;
+ i | v1 | j | v2 
+---+----+---+----
+ 1 | 10 | 2 | 10
+ 1 | 10 | 2 | 11
+ 1 | 11 | 2 | 10
+ 1 | 11 | 2 | 11
+(4 rows)
+
+SELECT *, count(*) OVER w FROM t1, t2
+WINDOW w AS (
+ PARTITION BY t1.i
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (A)
+ DEFINE
+ A AS v1 <= 11 AND v2 <= 11
+);
+ i | v1 | j | v2 | count 
+---+----+---+----+-------
+ 1 | 10 | 2 | 10 |     1
+ 1 | 10 | 2 | 11 |     1
+ 1 | 10 | 2 | 12 |     0
+ 1 | 11 | 2 | 10 |     1
+ 1 | 11 | 2 | 11 |     1
+ 1 | 11 | 2 | 12 |     0
+ 1 | 12 | 2 | 10 |     0
+ 1 | 12 | 2 | 11 |     0
+ 1 | 12 | 2 | 12 |     0
+(9 rows)
+
+-- WITH case
+WITH wstock AS (
+  SELECT * FROM stock WHERE tdate < '2023-07-08'
+)
+SELECT tdate, price,
+first_value(tdate) OVER w,
+count(*) OVER w
+ FROM wstock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+   tdate    | price | first_value | count 
+------------+-------+-------------+-------
+ 07-01-2023 |   100 | 07-01-2023  |     4
+ 07-02-2023 |   200 |             |     0
+ 07-03-2023 |   150 |             |     0
+ 07-04-2023 |   140 |             |     0
+ 07-05-2023 |   150 |             |     0
+ 07-06-2023 |    90 |             |     0
+ 07-07-2023 |   110 |             |     0
+ 07-01-2023 |    50 | 07-01-2023  |     4
+ 07-02-2023 |  2000 |             |     0
+ 07-03-2023 |  1500 |             |     0
+ 07-04-2023 |  1400 |             |     0
+ 07-05-2023 |  1500 |             |     0
+ 07-06-2023 |    60 |             |     0
+ 07-07-2023 |  1100 |             |     0
+(14 rows)
+
+--
+-- Error cases
+--
+-- row pattern definition variable name must not appear more than once
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ ORDER BY tdate
+ INITIAL
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price),
+  UP AS price > PREV(price)
+);
+ERROR:  syntax error at or near "ORDER"
+LINE 6:  ORDER BY tdate
+         ^
+-- pattern variable name must appear in DEFINE
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+ END)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+ERROR:  syntax error at or near "END"
+LINE 8:  PATTERN (START UP+ DOWN+ END)
+                                  ^
+-- FRAME must start at current row when row patttern recognition is used
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+ERROR:  FRAME must start at current row when row patttern recognition is used
+-- SEEK is not supported
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ AFTER MATCH SKIP TO NEXT ROW
+ SEEK
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+ERROR:  SEEK is not supported
+LINE 8:  SEEK
+         ^
+HINT:  Use INITIAL.
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index 675c567617..0ca2713c3d 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -98,7 +98,7 @@ test: publication subscription
 # Another group of parallel tests
 # select_views depends on create_view
 # ----------
-test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combocid tsearch tsdicts foreign_data window xmlmap functional_deps advisory_lock indirect_toast equivclass
+test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combocid tsearch tsdicts foreign_data window xmlmap functional_deps advisory_lock indirect_toast equivclass rpr
 
 # ----------
 # Another group of parallel tests (JSON related)
diff --git a/src/test/regress/sql/rpr.sql b/src/test/regress/sql/rpr.sql
new file mode 100644
index 0000000000..0a69cc0e11
--- /dev/null
+++ b/src/test/regress/sql/rpr.sql
@@ -0,0 +1,392 @@
+--
+-- Test for row pattern definition clause
+--
+
+CREATE TEMP TABLE stock (
+       company TEXT,
+       tdate DATE,
+       price INTEGER
+);
+INSERT INTO stock VALUES ('company1', '2023-07-01', 100);
+INSERT INTO stock VALUES ('company1', '2023-07-02', 200);
+INSERT INTO stock VALUES ('company1', '2023-07-03', 150);
+INSERT INTO stock VALUES ('company1', '2023-07-04', 140);
+INSERT INTO stock VALUES ('company1', '2023-07-05', 150);
+INSERT INTO stock VALUES ('company1', '2023-07-06', 90);
+INSERT INTO stock VALUES ('company1', '2023-07-07', 110);
+INSERT INTO stock VALUES ('company1', '2023-07-08', 130);
+INSERT INTO stock VALUES ('company1', '2023-07-09', 120);
+INSERT INTO stock VALUES ('company1', '2023-07-10', 130);
+INSERT INTO stock VALUES ('company2', '2023-07-01', 50);
+INSERT INTO stock VALUES ('company2', '2023-07-02', 2000);
+INSERT INTO stock VALUES ('company2', '2023-07-03', 1500);
+INSERT INTO stock VALUES ('company2', '2023-07-04', 1400);
+INSERT INTO stock VALUES ('company2', '2023-07-05', 1500);
+INSERT INTO stock VALUES ('company2', '2023-07-06', 60);
+INSERT INTO stock VALUES ('company2', '2023-07-07', 1100);
+INSERT INTO stock VALUES ('company2', '2023-07-08', 1300);
+INSERT INTO stock VALUES ('company2', '2023-07-09', 1200);
+INSERT INTO stock VALUES ('company2', '2023-07-10', 1300);
+
+SELECT * FROM stock;
+
+-- basic test using PREV
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w,
+ nth_value(tdate, 2) OVER w AS nth_second
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+
+-- basic test using PREV. UP appears twice
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w,
+ nth_value(tdate, 2) OVER w AS nth_second
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+ UP+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+
+-- basic test using PREV. Use '*'
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w,
+ nth_value(tdate, 2) OVER w AS nth_second
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP* DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+
+-- basic test with none greedy pattern
+SELECT company, tdate, price, count(*) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (A A A)
+ DEFINE
+  A AS price >= 140 AND price <= 150
+);
+
+-- last_value() should remain consistent
+SELECT company, tdate, price, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+
+-- omit "START" in DEFINE but it is ok because "START AS TRUE" is
+-- implicitly defined. per spec.
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w,
+ nth_value(tdate, 2) OVER w AS nth_second
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+
+-- the first row start with less than or equal to 100
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (LOWPRICE UP+ DOWN+)
+ DEFINE
+  LOWPRICE AS price <= 100,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+
+-- second row raises 120%
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (LOWPRICE UP+ DOWN+)
+ DEFINE
+  LOWPRICE AS price <= 100,
+  UP AS price > PREV(price) * 1.2,
+  DOWN AS price < PREV(price)
+);
+
+-- using NEXT
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UPDOWN)
+ DEFINE
+  START AS TRUE,
+  UPDOWN AS price > PREV(price) AND price > NEXT(price)
+);
+
+-- using AFTER MATCH SKIP TO NEXT ROW
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ AFTER MATCH SKIP TO NEXT ROW
+ INITIAL
+ PATTERN (START UPDOWN)
+ DEFINE
+  START AS TRUE,
+  UPDOWN AS price > PREV(price) AND price > NEXT(price)
+);
+
+-- match everything
+
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ AFTER MATCH SKIP PAST LAST ROW
+ INITIAL
+ PATTERN (A+)
+ DEFINE
+  A AS TRUE
+);
+
+-- backtracking with reclassification of rows
+-- using AFTER MATCH SKIP PAST LAST ROW
+SELECT company, tdate, price, first_value(tdate) OVER w, last_value(tdate) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ AFTER MATCH SKIP PAST LAST ROW
+ INITIAL
+ PATTERN (A+ B+)
+ DEFINE
+  A AS price > 100,
+  B AS price > 100
+);
+
+-- backtracking with reclassification of rows
+-- using AFTER MATCH SKIP TO NEXT ROW
+SELECT company, tdate, price, first_value(tdate) OVER w, last_value(tdate) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ AFTER MATCH SKIP TO NEXT ROW
+ INITIAL
+ PATTERN (A+ B+)
+ DEFINE
+  A AS price > 100,
+  B AS price > 100
+);
+
+-- ROWS BETWEEN CURRENT ROW AND offset FOLLOWING
+SELECT company, tdate, price, first_value(tdate) OVER w, last_value(tdate) OVER w,
+ count(*) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND 2 FOLLOWING
+ AFTER MATCH SKIP PAST LAST ROW
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+
+--
+-- Aggregates
+--
+
+-- using AFTER MATCH SKIP PAST LAST ROW
+SELECT company, tdate, price,
+ first_value(price) OVER w,
+ last_value(price) OVER w,
+ max(price) OVER w,
+ min(price) OVER w,
+ sum(price) OVER w,
+ avg(price) OVER w,
+ count(price) OVER w
+FROM stock
+WINDOW w AS (
+PARTITION BY company
+ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+AFTER MATCH SKIP PAST LAST ROW
+INITIAL
+PATTERN (START UP+ DOWN+)
+DEFINE
+START AS TRUE,
+UP AS price > PREV(price),
+DOWN AS price < PREV(price)
+);
+
+-- using AFTER MATCH SKIP TO NEXT ROW
+SELECT company, tdate, price,
+ first_value(price) OVER w,
+ last_value(price) OVER w,
+ max(price) OVER w,
+ min(price) OVER w,
+ sum(price) OVER w,
+ avg(price) OVER w,
+ count(price) OVER w
+FROM stock
+WINDOW w AS (
+PARTITION BY company
+ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+AFTER MATCH SKIP TO NEXT ROW
+INITIAL
+PATTERN (START UP+ DOWN+)
+DEFINE
+START AS TRUE,
+UP AS price > PREV(price),
+DOWN AS price < PREV(price)
+);
+
+-- JOIN case
+CREATE TEMP TABLE t1 (i int, v1 int);
+CREATE TEMP TABLE t2 (j int, v2 int);
+INSERT INTO t1 VALUES(1,10);
+INSERT INTO t1 VALUES(1,11);
+INSERT INTO t1 VALUES(1,12);
+INSERT INTO t2 VALUES(2,10);
+INSERT INTO t2 VALUES(2,11);
+INSERT INTO t2 VALUES(2,12);
+
+SELECT * FROM t1, t2 WHERE t1.v1 <= 11 AND t2.v2 <= 11;
+
+SELECT *, count(*) OVER w FROM t1, t2
+WINDOW w AS (
+ PARTITION BY t1.i
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (A)
+ DEFINE
+ A AS v1 <= 11 AND v2 <= 11
+);
+
+-- WITH case
+WITH wstock AS (
+  SELECT * FROM stock WHERE tdate < '2023-07-08'
+)
+SELECT tdate, price,
+first_value(tdate) OVER w,
+count(*) OVER w
+ FROM wstock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+
+--
+-- Error cases
+--
+
+-- row pattern definition variable name must not appear more than once
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ ORDER BY tdate
+ INITIAL
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price),
+  UP AS price > PREV(price)
+);
+
+-- pattern variable name must appear in DEFINE
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+ END)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+
+-- FRAME must start at current row when row patttern recognition is used
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+ INITIAL
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
+
+-- SEEK is not supported
+SELECT company, tdate, price, first_value(price) OVER w, last_value(price) OVER w
+ FROM stock
+ WINDOW w AS (
+ PARTITION BY company
+ ORDER BY tdate
+ ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+ AFTER MATCH SKIP TO NEXT ROW
+ SEEK
+ PATTERN (START UP+ DOWN+)
+ DEFINE
+  START AS TRUE,
+  UP AS price > PREV(price),
+  DOWN AS price < PREV(price)
+);
-- 
2.25.1


----Next_Part(Sun_Apr_28_20_28_26_2024_444)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="v17-0008-Allow-to-print-raw-parse-tree.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread

* [PATCH v13a 7/9] ci: get build/test concurrency from environment
@ 2026-06-10 18:49 Andres Freund <[email protected]>
  0 siblings, 0 replies; 156+ messages in thread

From: Andres Freund @ 2026-06-10 18:49 UTC (permalink / raw)

Previously we hardcoded the amount of concurrency in the CI definition. This
practice IIRC originated from a) using make, where the concurrency is not
sourced from the current system b) early cirrus-ci macos runners, which slowed
down a lot with full concurrency.

Hardcoding the concurrency is problematic as e.g. private repositories have
lower concurrency and larger github runners have more cores.

For most of the jobs we can rely on just using meson's and ninja's logic for
getting the current core count. For the autoconf we can use the nproc helper.

To make this at least somewhat understandable, I added the existing sysinfo
step to show the number of cores (and made it a bit easier to understand the
information already printed).

Discussion: https://postgr.es/m/a2ejn7lfqolutzz7kozalbhy3bixdrujb4buc3pgbtlk4am2ba@wbv6v7riia33
---
 .github/workflows/pg-ci.yml | 47 +++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index ab7950f6f14..9384ec0d902 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -40,14 +40,6 @@ env:
   # concurrent jobs and retrying older runs have a chance of working.
   CLONE_DEPTH: 500
 
-  # At the moment all jobs use 4vcore runners, and none seems to benefit from
-  # increasing concurrency further.
-  BUILD_JOBS: 4
-
-  # It's possible that some jobs benefit from an increased test concurrency,
-  # but a default of 4 is a safe bet. Individual jobs can override.
-  TEST_JOBS: 4
-
   CCACHE_MAXSIZE: "250M"
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
 
@@ -215,10 +207,19 @@ jobs:
       - &nix_sysinfo_step
         name: sysinfo
         run: |
-          id
-          uname -a
+          echo -n "Number of cores: "
+          nproc 2> /dev/null || sysctl hw.physicalcpu hw.logicalcpu
+
+          echo "id: $(id)"
+          echo "uname: $(uname -a)"
+
+          echo ::group::ulimit
           ulimit -a -H && ulimit -a -S
+          echo ::endgroup::
+
+          echo ::group::env
           env
+          echo ::endgroup::
 
       - name: Parse ci-os-only
         id: os
@@ -352,7 +353,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: &ninja_build_cmd |
-          ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
       # Decide if it's worth uploading a new version of the ccache cache. If
@@ -405,7 +406,7 @@ jobs:
           meson test ${{env.MTEST_ARGS}} --suite setup --logbase setup || exit 1
           echo ::endgroup::
 
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --no-suite setup ${{env.MTEST_TARGET}}
+          meson test ${{env.MTEST_ARGS}} --no-suite setup ${{env.MTEST_TARGET}}
 
       - &linux_collect_cores_step
         name: Core backtraces
@@ -511,7 +512,7 @@ jobs:
       - name: Build
         shell: *su_postgres_shell
         run: |
-          make -s -j${BUILD_JOBS} world-bin
+          make -s -j$(nproc) world-bin
 
       - *ccache_decide_save_step
       - *ccache_save_step
@@ -519,7 +520,7 @@ jobs:
       - name: Test world
         shell: *su_postgres_shell
         run: |
-          make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+          make -s ${CHECK} ${CHECKFLAGS} -j$(nproc)
 
       - *linux_collect_cores_step
       - *upload_logs_step
@@ -611,7 +612,7 @@ jobs:
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
 
           # Run the tests supporting running against an already running
-          meson test ${{env.MTEST_ARGS}} --num-processes ${{env.TEST_JOBS}} --setup running
+          meson test ${{env.MTEST_ARGS}} --setup running
 
           build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
 
@@ -893,9 +894,19 @@ jobs:
 
       - name: Sysinfo
         run: |
+          echo Number of cores:
+          bash -c nproc
+
+          echo codepage:
           chcp
+
+          echo ::group::systeminfo
           systeminfo
+          echo ::endgroup::
+
+          echo ::group::env
           set
+          echo ::endgroup::
 
       # The TAP tests build an initdb template under build/tmp_install and
       # then `robocopy` it into per-test data directories. Robocopy with the
@@ -1198,7 +1209,7 @@ jobs:
           meson setup -Dwerror=true $CONF build-meson
           echo "::endgroup::"
 
-          ninja -C build-meson --quiet -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
+          ninja -C build-meson --quiet ${{env.MBUILD_TARGET}}
 
       # gcc, cassert on, dtrace off
       - name: gcc warnings + (cassert)
@@ -1266,8 +1277,8 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
           echo "::endgroup::"
 
-          make -s -j${{env.BUILD_JOBS}} clean
-          make -s -j${{env.BUILD_JOBS}} -k ${{env.CHECKFLAGS}} \
+          make -s -j$(nproc) clean
+          make -s -j$(nproc) -k ${{env.CHECKFLAGS}} \
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
-- 
2.54.0.450.g9ac3f193c0


--mr2uqtieh2e2xvha
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="v13a-0008-ci-Make-runs-on-overridable.patch"



^ permalink  raw  reply  [nested|flat] 156+ messages in thread