agora inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH v5 04/10] Allow to use multiple shared memory mappings
436+ messages / 2 participants
[nested] [flat]

* [PATCH v5 04/10] Allow to use multiple shared memory mappings
@ 2025-02-28 18:54  Dmitrii Dolgov <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Dmitrii Dolgov @ 2025-02-28 18:54 UTC (permalink / raw)

Currently all the work with shared memory is done via a single anonymous
memory mapping, which limits ways how the shared memory could be organized.

Introduce possibility to allocate multiple shared memory mappings, where
a single mapping is associated with a specified shared memory segment.
There is only fixed amount of available segments, currently only one
main shared memory segment is allocated. A new shared memory API is
introduces, extended with a segment as a new parameter. As a path of
least resistance, the original API is kept in place, utilizing the main
shared memory segment.
---
 src/backend/port/posix_sema.c     |   4 +-
 src/backend/port/sysv_sema.c      |   4 +-
 src/backend/port/sysv_shmem.c     | 138 +++++++++++++++++++---------
 src/backend/port/win32_sema.c     |   2 +-
 src/backend/storage/ipc/ipc.c     |   4 +-
 src/backend/storage/ipc/ipci.c    |  63 +++++++------
 src/backend/storage/ipc/shmem.c   | 148 +++++++++++++++++++++---------
 src/backend/storage/lmgr/lwlock.c |  13 ++-
 src/include/storage/ipc.h         |   2 +-
 src/include/storage/pg_sema.h     |   2 +-
 src/include/storage/pg_shmem.h    |  18 ++++
 src/include/storage/shmem.h       |  12 +++
 12 files changed, 284 insertions(+), 126 deletions(-)

diff --git a/src/backend/port/posix_sema.c b/src/backend/port/posix_sema.c
index 269c7460817..401e1113fa1 100644
--- a/src/backend/port/posix_sema.c
+++ b/src/backend/port/posix_sema.c
@@ -193,7 +193,7 @@ PGSemaphoreShmemSize(int maxSemas)
  * we don't have to expose the counters to other processes.)
  */
 void
-PGReserveSemaphores(int maxSemas)
+PGReserveSemaphores(int maxSemas, int shmem_segment)
 {
 	struct stat statbuf;
 
@@ -220,7 +220,7 @@ PGReserveSemaphores(int maxSemas)
 	 * ShmemAlloc() won't be ready yet.
 	 */
 	sharedSemas = (PGSemaphore)
-		ShmemAllocUnlocked(PGSemaphoreShmemSize(maxSemas));
+		ShmemAllocUnlockedInSegment(PGSemaphoreShmemSize(maxSemas), shmem_segment);
 #endif
 
 	numSems = 0;
diff --git a/src/backend/port/sysv_sema.c b/src/backend/port/sysv_sema.c
index 423b2b4f9d6..4ce2cfb662b 100644
--- a/src/backend/port/sysv_sema.c
+++ b/src/backend/port/sysv_sema.c
@@ -307,7 +307,7 @@ PGSemaphoreShmemSize(int maxSemas)
  * have clobbered.)
  */
 void
-PGReserveSemaphores(int maxSemas)
+PGReserveSemaphores(int maxSemas, int shmem_segment)
 {
 	struct stat statbuf;
 
@@ -328,7 +328,7 @@ PGReserveSemaphores(int maxSemas)
 	 * ShmemAlloc() won't be ready yet.
 	 */
 	sharedSemas = (PGSemaphore)
-		ShmemAllocUnlocked(PGSemaphoreShmemSize(maxSemas));
+		ShmemAllocUnlockedInSegment(PGSemaphoreShmemSize(maxSemas), shmem_segment);
 	numSharedSemas = 0;
 	maxSharedSemas = maxSemas;
 
diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 197926d44f6..56af0231d24 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -94,8 +94,19 @@ typedef enum
 unsigned long UsedShmemSegID = 0;
 void	   *UsedShmemSegAddr = NULL;
 
-static Size AnonymousShmemSize;
-static void *AnonymousShmem = NULL;
+typedef struct AnonymousMapping
+{
+	int shmem_segment;
+	Size shmem_size; 			/* Size of the mapping */
+	Pointer shmem; 				/* Pointer to the start of the mapped memory */
+	Pointer seg_addr; 			/* SysV shared memory for the header */
+	unsigned long seg_id; 		/* IPC key */
+} AnonymousMapping;
+
+static AnonymousMapping Mappings[ANON_MAPPINGS];
+
+/* Keeps track of used mapping segments */
+static int next_free_segment = 0;
 
 static void *InternalIpcMemoryCreate(IpcMemoryKey memKey, Size size);
 static void IpcMemoryDetach(int status, Datum shmaddr);
@@ -104,6 +115,28 @@ static IpcMemoryState PGSharedMemoryAttach(IpcMemoryId shmId,
 										   void *attachAt,
 										   PGShmemHeader **addr);
 
+static const char*
+MappingName(int shmem_segment)
+{
+	switch (shmem_segment)
+	{
+		case MAIN_SHMEM_SEGMENT:
+			return "main";
+		default:
+			return "unknown";
+	}
+}
+
+static void
+DebugMappings()
+{
+	for(int i = 0; i < next_free_segment; i++)
+	{
+		AnonymousMapping m = Mappings[i];
+		elog(DEBUG1, "Mapping[%s]: addr %p, size %zu",
+			 MappingName(i), m.shmem, m.shmem_size);
+	}
+}
 
 /*
  *	InternalIpcMemoryCreate(memKey, size)
@@ -591,14 +624,13 @@ check_huge_page_size(int *newval, void **extra, GucSource source)
 /*
  * Creates an anonymous mmap()ed shared memory segment.
  *
- * Pass the requested size in *size.  This function will modify *size to the
- * actual size of the allocation, if it ends up allocating a segment that is
- * larger than requested.
+ * This function will modify mapping size to the actual size of the allocation,
+ * if it ends up allocating a segment that is larger than requested.
  */
-static void *
-CreateAnonymousSegment(Size *size)
+static void
+CreateAnonymousSegment(AnonymousMapping *mapping)
 {
-	Size		allocsize = *size;
+	Size		allocsize = mapping->shmem_size;
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
@@ -623,8 +655,11 @@ CreateAnonymousSegment(Size *size)
 				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
-			elog(DEBUG1, "mmap(%zu) with MAP_HUGETLB failed, huge pages disabled: %m",
-				 allocsize);
+		{
+			DebugMappings();
+			elog(DEBUG1, "segment[%s]: mmap(%zu) with MAP_HUGETLB failed, huge pages disabled: %m",
+				 MappingName(mapping->shmem_segment), allocsize);
+		}
 	}
 #endif
 
@@ -642,7 +677,7 @@ CreateAnonymousSegment(Size *size)
 		 * Use the original size, not the rounded-up value, when falling back
 		 * to non-huge pages.
 		 */
-		allocsize = *size;
+		allocsize = mapping->shmem_size;
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
 				   PG_MMAP_FLAGS, -1, 0);
 		mmap_errno = errno;
@@ -651,8 +686,10 @@ CreateAnonymousSegment(Size *size)
 	if (ptr == MAP_FAILED)
 	{
 		errno = mmap_errno;
+		DebugMappings();
 		ereport(FATAL,
-				(errmsg("could not map anonymous shared memory: %m"),
+				(errmsg("segment[%s]: could not map anonymous shared memory: %m",
+						MappingName(mapping->shmem_segment)),
 				 (mmap_errno == ENOMEM) ?
 				 errhint("This error usually means that PostgreSQL's request "
 						 "for a shared memory segment exceeded available memory, "
@@ -663,8 +700,8 @@ CreateAnonymousSegment(Size *size)
 						 allocsize) : 0));
 	}
 
-	*size = allocsize;
-	return ptr;
+	mapping->shmem = ptr;
+	mapping->shmem_size = allocsize;
 }
 
 /*
@@ -674,13 +711,18 @@ CreateAnonymousSegment(Size *size)
 static void
 AnonymousShmemDetach(int status, Datum arg)
 {
-	/* Release anonymous shared memory block, if any. */
-	if (AnonymousShmem != NULL)
+	for(int i = 0; i < next_free_segment; i++)
 	{
-		if (munmap(AnonymousShmem, AnonymousShmemSize) < 0)
-			elog(LOG, "munmap(%p, %zu) failed: %m",
-				 AnonymousShmem, AnonymousShmemSize);
-		AnonymousShmem = NULL;
+		AnonymousMapping m = Mappings[i];
+
+		/* Release anonymous shared memory block, if any. */
+		if (m.shmem != NULL)
+		{
+			if (munmap(m.shmem, m.shmem_size) < 0)
+				elog(LOG, "munmap(%p, %zu) failed: %m",
+					 m.shmem, m.shmem_size);
+			m.shmem = NULL;
+		}
 	}
 }
 
@@ -705,6 +747,7 @@ PGSharedMemoryCreate(Size size,
 	PGShmemHeader *hdr;
 	struct stat statbuf;
 	Size		sysvsize;
+	AnonymousMapping *mapping = &Mappings[next_free_segment];
 
 	/*
 	 * We use the data directory's ID info (inode and device numbers) to
@@ -733,11 +776,15 @@ PGSharedMemoryCreate(Size size,
 
 	/* Room for a header? */
 	Assert(size > MAXALIGN(sizeof(PGShmemHeader)));
+	mapping->shmem_size = size;
+	mapping->shmem_segment = next_free_segment;
 
 	if (shared_memory_type == SHMEM_TYPE_MMAP)
 	{
-		AnonymousShmem = CreateAnonymousSegment(&size);
-		AnonymousShmemSize = size;
+		/* On success, mapping data will be modified. */
+		CreateAnonymousSegment(mapping);
+
+		next_free_segment++;
 
 		/* Register on-exit routine to unmap the anonymous segment */
 		on_shmem_exit(AnonymousShmemDetach, (Datum) 0);
@@ -760,7 +807,7 @@ PGSharedMemoryCreate(Size size,
 	 * loop simultaneously.  (CreateDataDirLockFile() does not entirely ensure
 	 * that, but prefer fixing it over coping here.)
 	 */
-	NextShmemSegID = statbuf.st_ino;
+	NextShmemSegID = statbuf.st_ino + next_free_segment;
 
 	for (;;)
 	{
@@ -852,13 +899,13 @@ PGSharedMemoryCreate(Size size,
 	/*
 	 * Initialize space allocation status for segment.
 	 */
-	hdr->totalsize = size;
+	hdr->totalsize = mapping->shmem_size;
 	hdr->freeoffset = MAXALIGN(sizeof(PGShmemHeader));
 	*shim = hdr;
 
 	/* Save info for possible future use */
-	UsedShmemSegAddr = memAddress;
-	UsedShmemSegID = (unsigned long) NextShmemSegID;
+	mapping->seg_addr = memAddress;
+	mapping->seg_id = (unsigned long) NextShmemSegID;
 
 	/*
 	 * If AnonymousShmem is NULL here, then we're not using anonymous shared
@@ -866,10 +913,10 @@ PGSharedMemoryCreate(Size size,
 	 * block. Otherwise, the System V shared memory block is only a shim, and
 	 * we must return a pointer to the real block.
 	 */
-	if (AnonymousShmem == NULL)
+	if (mapping->shmem == NULL)
 		return hdr;
-	memcpy(AnonymousShmem, hdr, sizeof(PGShmemHeader));
-	return (PGShmemHeader *) AnonymousShmem;
+	memcpy(mapping->shmem, hdr, sizeof(PGShmemHeader));
+	return (PGShmemHeader *) mapping->shmem;
 }
 
 #ifdef EXEC_BACKEND
@@ -969,23 +1016,28 @@ PGSharedMemoryNoReAttach(void)
 void
 PGSharedMemoryDetach(void)
 {
-	if (UsedShmemSegAddr != NULL)
+	for(int i = 0; i < next_free_segment; i++)
 	{
-		if ((shmdt(UsedShmemSegAddr) < 0)
+		AnonymousMapping m = Mappings[i];
+
+		if (m.seg_addr != NULL)
+		{
+			if ((shmdt(m.seg_addr) < 0)
 #if defined(EXEC_BACKEND) && defined(__CYGWIN__)
-		/* Work-around for cygipc exec bug */
-			&& shmdt(NULL) < 0
+			/* Work-around for cygipc exec bug */
+				&& shmdt(NULL) < 0
 #endif
-			)
-			elog(LOG, "shmdt(%p) failed: %m", UsedShmemSegAddr);
-		UsedShmemSegAddr = NULL;
-	}
+				)
+				elog(LOG, "shmdt(%p) failed: %m", m.seg_addr);
+			m.seg_addr = NULL;
+		}
 
-	if (AnonymousShmem != NULL)
-	{
-		if (munmap(AnonymousShmem, AnonymousShmemSize) < 0)
-			elog(LOG, "munmap(%p, %zu) failed: %m",
-				 AnonymousShmem, AnonymousShmemSize);
-		AnonymousShmem = NULL;
+		if (m.shmem != NULL)
+		{
+			if (munmap(m.shmem, m.shmem_size) < 0)
+				elog(LOG, "munmap(%p, %zu) failed: %m",
+					 m.shmem, m.shmem_size);
+			m.shmem = NULL;
+		}
 	}
 }
diff --git a/src/backend/port/win32_sema.c b/src/backend/port/win32_sema.c
index 5854ad1f54d..e7365ff8060 100644
--- a/src/backend/port/win32_sema.c
+++ b/src/backend/port/win32_sema.c
@@ -44,7 +44,7 @@ PGSemaphoreShmemSize(int maxSemas)
  * process exits.
  */
 void
-PGReserveSemaphores(int maxSemas)
+PGReserveSemaphores(int maxSemas, int shmem_segment)
 {
 	mySemSet = (HANDLE *) malloc(maxSemas * sizeof(HANDLE));
 	if (mySemSet == NULL)
diff --git a/src/backend/storage/ipc/ipc.c b/src/backend/storage/ipc/ipc.c
index 567739b5be9..5b55bec8d9d 100644
--- a/src/backend/storage/ipc/ipc.c
+++ b/src/backend/storage/ipc/ipc.c
@@ -61,6 +61,8 @@ static void proc_exit_prepare(int code);
  * but provide some additional features we need --- in particular,
  * we want to register callbacks to invoke when we are disconnecting
  * from a broken shared-memory context but not exiting the postmaster.
+ * Maximum number of such exit callbacks depends on the number of shared
+ * segments.
  *
  * Callback functions can take zero, one, or two args: the first passed
  * arg is the integer exitcode, the second is the Datum supplied when
@@ -68,7 +70,7 @@ static void proc_exit_prepare(int code);
  * ----------------------------------------------------------------
  */
 
-#define MAX_ON_EXITS 20
+#define MAX_ON_EXITS 40
 
 struct ONEXIT
 {
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 2fa045e6b0f..8b38e985327 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -86,7 +86,7 @@ RequestAddinShmemSpace(Size size)
  * required.
  */
 Size
-CalculateShmemSize(int *num_semaphores)
+CalculateShmemSize(int *num_semaphores, int shmem_segment)
 {
 	Size		size;
 	int			numSemas;
@@ -206,33 +206,38 @@ CreateSharedMemoryAndSemaphores(void)
 
 	Assert(!IsUnderPostmaster);
 
-	/* Compute the size of the shared-memory block */
-	size = CalculateShmemSize(&numSemas);
-	elog(DEBUG3, "invoking IpcMemoryCreate(size=%zu)", size);
-
-	/*
-	 * Create the shmem segment
-	 */
-	seghdr = PGSharedMemoryCreate(size, &shim);
-
-	/*
-	 * Make sure that huge pages are never reported as "unknown" while the
-	 * server is running.
-	 */
-	Assert(strcmp("unknown",
-				  GetConfigOption("huge_pages_status", false, false)) != 0);
-
-	InitShmemAccess(seghdr);
-
-	/*
-	 * Create semaphores
-	 */
-	PGReserveSemaphores(numSemas);
-
-	/*
-	 * Set up shared memory allocation mechanism
-	 */
-	InitShmemAllocation();
+	for(int segment = 0; segment < ANON_MAPPINGS; segment++)
+	{
+		/* Compute the size of the shared-memory block */
+		size = CalculateShmemSize(&numSemas, segment);
+		elog(DEBUG3, "invoking IpcMemoryCreate(size=%zu)", size);
+
+		/*
+		 * Create the shmem segment.
+		 *
+		 * XXX: Do multiple shims are needed, one per segment?
+		 */
+		seghdr = PGSharedMemoryCreate(size, &shim);
+
+		/*
+		 * Make sure that huge pages are never reported as "unknown" while the
+		 * server is running.
+		 */
+		Assert(strcmp("unknown",
+					  GetConfigOption("huge_pages_status", false, false)) != 0);
+
+		InitShmemAccessInSegment(seghdr, segment);
+
+		/*
+		 * Create semaphores
+		 */
+		PGReserveSemaphores(numSemas, segment);
+
+		/*
+		 * Set up shared memory allocation mechanism
+		 */
+		InitShmemAllocationInSegment(segment);
+	}
 
 	/* Initialize subsystems */
 	CreateOrAttachShmemStructs();
@@ -363,7 +368,7 @@ InitializeShmemGUCs(void)
 	/*
 	 * Calculate the shared memory size and round up to the nearest megabyte.
 	 */
-	size_b = CalculateShmemSize(&num_semas);
+	size_b = CalculateShmemSize(&num_semas, MAIN_SHMEM_SEGMENT);
 	size_mb = add_size(size_b, (1024 * 1024) - 1) / (1024 * 1024);
 	sprintf(buf, "%zu", size_mb);
 	SetConfigOption("shared_memory_size", buf,
diff --git a/src/backend/storage/ipc/shmem.c b/src/backend/storage/ipc/shmem.c
index c9ae3b45b76..72255a1c5ca 100644
--- a/src/backend/storage/ipc/shmem.c
+++ b/src/backend/storage/ipc/shmem.c
@@ -76,19 +76,19 @@
 #include "utils/builtins.h"
 
 static void *ShmemAllocRaw(Size size, Size *allocated_size);
+static void *ShmemAllocRawInSegment(Size size, Size *allocated_size,
+								 int shmem_segment);
 
 /* shared memory global variables */
 
-static PGShmemHeader *ShmemSegHdr;	/* shared mem segment header */
+ShmemSegment Segments[ANON_MAPPINGS];
 
-static void *ShmemBase;			/* start address of shared memory */
-
-static void *ShmemEnd;			/* end+1 address of shared memory */
-
-slock_t    *ShmemLock;			/* spinlock for shared memory and LWLock
-								 * allocation */
-
-static HTAB *ShmemIndex = NULL; /* primary index hashtable for shmem */
+/*
+ * Primary index hashtable for shmem, for simplicity we use a single for all
+ * shared memory segments. There can be performance consequences of that, and
+ * an alternative option would be to have one index per shared memory segments.
+ */
+static HTAB *ShmemIndex = NULL;
 
 /* To get reliable results for NUMA inquiry we need to "touch pages" once */
 static bool firstNumaTouch = true;
@@ -101,9 +101,17 @@ Datum		pg_numa_available(PG_FUNCTION_ARGS);
 void
 InitShmemAccess(PGShmemHeader *seghdr)
 {
-	ShmemSegHdr = seghdr;
-	ShmemBase = seghdr;
-	ShmemEnd = (char *) ShmemBase + seghdr->totalsize;
+	InitShmemAccessInSegment(seghdr, MAIN_SHMEM_SEGMENT);
+}
+
+void
+InitShmemAccessInSegment(PGShmemHeader *seghdr, int shmem_segment)
+{
+	PGShmemHeader *shmhdr = (PGShmemHeader *) seghdr;
+	ShmemSegment *seg = &Segments[shmem_segment];
+	seg->ShmemSegHdr = shmhdr;
+	seg->ShmemBase = (void *) shmhdr;
+	seg->ShmemEnd = (char *) seg->ShmemBase + shmhdr->totalsize;
 }
 
 /*
@@ -114,7 +122,13 @@ InitShmemAccess(PGShmemHeader *seghdr)
 void
 InitShmemAllocation(void)
 {
-	PGShmemHeader *shmhdr = ShmemSegHdr;
+	InitShmemAllocationInSegment(MAIN_SHMEM_SEGMENT);
+}
+
+void
+InitShmemAllocationInSegment(int shmem_segment)
+{
+	PGShmemHeader *shmhdr = Segments[shmem_segment].ShmemSegHdr;
 	char	   *aligned;
 
 	Assert(shmhdr != NULL);
@@ -123,9 +137,9 @@ InitShmemAllocation(void)
 	 * Initialize the spinlock used by ShmemAlloc.  We must use
 	 * ShmemAllocUnlocked, since obviously ShmemAlloc can't be called yet.
 	 */
-	ShmemLock = (slock_t *) ShmemAllocUnlocked(sizeof(slock_t));
+	Segments[shmem_segment].ShmemLock = (slock_t *) ShmemAllocUnlockedInSegment(sizeof(slock_t), shmem_segment);
 
-	SpinLockInit(ShmemLock);
+	SpinLockInit(Segments[shmem_segment].ShmemLock);
 
 	/*
 	 * Allocations after this point should go through ShmemAlloc, which
@@ -150,11 +164,17 @@ InitShmemAllocation(void)
  */
 void *
 ShmemAlloc(Size size)
+{
+	return ShmemAllocInSegment(size, MAIN_SHMEM_SEGMENT);
+}
+
+void *
+ShmemAllocInSegment(Size size, int shmem_segment)
 {
 	void	   *newSpace;
 	Size		allocated_size;
 
-	newSpace = ShmemAllocRaw(size, &allocated_size);
+	newSpace = ShmemAllocRawInSegment(size, &allocated_size, shmem_segment);
 	if (!newSpace)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -184,6 +204,12 @@ ShmemAllocNoError(Size size)
  */
 static void *
 ShmemAllocRaw(Size size, Size *allocated_size)
+{
+	return ShmemAllocRawInSegment(size, allocated_size, MAIN_SHMEM_SEGMENT);
+}
+
+static void *
+ShmemAllocRawInSegment(Size size, Size *allocated_size, int shmem_segment)
 {
 	Size		newStart;
 	Size		newFree;
@@ -203,22 +229,22 @@ ShmemAllocRaw(Size size, Size *allocated_size)
 	size = CACHELINEALIGN(size);
 	*allocated_size = size;
 
-	Assert(ShmemSegHdr != NULL);
+	Assert(Segments[shmem_segment].ShmemSegHdr != NULL);
 
-	SpinLockAcquire(ShmemLock);
+	SpinLockAcquire(Segments[shmem_segment].ShmemLock);
 
-	newStart = ShmemSegHdr->freeoffset;
+	newStart = Segments[shmem_segment].ShmemSegHdr->freeoffset;
 
 	newFree = newStart + size;
-	if (newFree <= ShmemSegHdr->totalsize)
+	if (newFree <= Segments[shmem_segment].ShmemSegHdr->totalsize)
 	{
-		newSpace = (char *) ShmemBase + newStart;
-		ShmemSegHdr->freeoffset = newFree;
+		newSpace = (char *) Segments[shmem_segment].ShmemBase + newStart;
+		Segments[shmem_segment].ShmemSegHdr->freeoffset = newFree;
 	}
 	else
 		newSpace = NULL;
 
-	SpinLockRelease(ShmemLock);
+	SpinLockRelease(Segments[shmem_segment].ShmemLock);
 
 	/* note this assert is okay with newSpace == NULL */
 	Assert(newSpace == (void *) CACHELINEALIGN(newSpace));
@@ -236,6 +262,12 @@ ShmemAllocRaw(Size size, Size *allocated_size)
  */
 void *
 ShmemAllocUnlocked(Size size)
+{
+	return ShmemAllocUnlockedInSegment(size, MAIN_SHMEM_SEGMENT);
+}
+
+void *
+ShmemAllocUnlockedInSegment(Size size, int shmem_segment)
 {
 	Size		newStart;
 	Size		newFree;
@@ -246,19 +278,19 @@ ShmemAllocUnlocked(Size size)
 	 */
 	size = MAXALIGN(size);
 
-	Assert(ShmemSegHdr != NULL);
+	Assert(Segments[shmem_segment].ShmemSegHdr != NULL);
 
-	newStart = ShmemSegHdr->freeoffset;
+	newStart = Segments[shmem_segment].ShmemSegHdr->freeoffset;
 
 	newFree = newStart + size;
-	if (newFree > ShmemSegHdr->totalsize)
+	if (newFree > Segments[shmem_segment].ShmemSegHdr->totalsize)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of shared memory (%zu bytes requested)",
 						size)));
-	ShmemSegHdr->freeoffset = newFree;
+	Segments[shmem_segment].ShmemSegHdr->freeoffset = newFree;
 
-	newSpace = (char *) ShmemBase + newStart;
+	newSpace = (char *) Segments[shmem_segment].ShmemBase + newStart;
 
 	Assert(newSpace == (void *) MAXALIGN(newSpace));
 
@@ -273,7 +305,13 @@ ShmemAllocUnlocked(Size size)
 bool
 ShmemAddrIsValid(const void *addr)
 {
-	return (addr >= ShmemBase) && (addr < ShmemEnd);
+	return ShmemAddrIsValidInSegment(addr, MAIN_SHMEM_SEGMENT);
+}
+
+bool
+ShmemAddrIsValidInSegment(const void *addr, int shmem_segment)
+{
+	return (addr >= Segments[shmem_segment].ShmemBase) && (addr < Segments[shmem_segment].ShmemEnd);
 }
 
 /*
@@ -334,6 +372,18 @@ ShmemInitHash(const char *name,		/* table string name for shmem index */
 			  long max_size,	/* max size of the table */
 			  HASHCTL *infoP,	/* info about key and bucket size */
 			  int hash_flags)	/* info about infoP */
+{
+	return ShmemInitHashInSegment(name, init_size, max_size, infoP, hash_flags,
+							   MAIN_SHMEM_SEGMENT);
+}
+
+HTAB *
+ShmemInitHashInSegment(const char *name,		/* table string name for shmem index */
+			  long init_size,		/* initial table size */
+			  long max_size,		/* max size of the table */
+			  HASHCTL *infoP,		/* info about key and bucket size */
+			  int hash_flags,		/* info about infoP */
+			  int shmem_segment) 	/* in which segment to keep the table */
 {
 	bool		found;
 	void	   *location;
@@ -350,9 +400,9 @@ ShmemInitHash(const char *name,		/* table string name for shmem index */
 	hash_flags |= HASH_SHARED_MEM | HASH_ALLOC | HASH_DIRSIZE;
 
 	/* look it up in the shmem index */
-	location = ShmemInitStruct(name,
+	location = ShmemInitStructInSegment(name,
 							   hash_get_shared_size(infoP, hash_flags),
-							   &found);
+							   &found, shmem_segment);
 
 	/*
 	 * if it already exists, attach to it rather than allocate and initialize
@@ -385,6 +435,13 @@ ShmemInitHash(const char *name,		/* table string name for shmem index */
  */
 void *
 ShmemInitStruct(const char *name, Size size, bool *foundPtr)
+{
+	return ShmemInitStructInSegment(name, size, foundPtr, MAIN_SHMEM_SEGMENT);
+}
+
+void *
+ShmemInitStructInSegment(const char *name, Size size, bool *foundPtr,
+					  int shmem_segment)
 {
 	ShmemIndexEnt *result;
 	void	   *structPtr;
@@ -393,7 +450,7 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
 
 	if (!ShmemIndex)
 	{
-		PGShmemHeader *shmemseghdr = ShmemSegHdr;
+		PGShmemHeader *shmemseghdr = Segments[shmem_segment].ShmemSegHdr;
 
 		/* Must be trying to create/attach to ShmemIndex itself */
 		Assert(strcmp(name, "ShmemIndex") == 0);
@@ -416,7 +473,7 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
 			 * process can be accessing shared memory yet.
 			 */
 			Assert(shmemseghdr->index == NULL);
-			structPtr = ShmemAlloc(size);
+			structPtr = ShmemAllocInSegment(size, shmem_segment);
 			shmemseghdr->index = structPtr;
 			*foundPtr = false;
 		}
@@ -433,8 +490,8 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
 		LWLockRelease(ShmemIndexLock);
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
-				 errmsg("could not create ShmemIndex entry for data structure \"%s\"",
-						name)));
+				 errmsg("could not create ShmemIndex entry for data structure \"%s\" in segment %d",
+						name, shmem_segment)));
 	}
 
 	if (*foundPtr)
@@ -459,7 +516,7 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
 		Size		allocated_size;
 
 		/* It isn't in the table yet. allocate and initialize it */
-		structPtr = ShmemAllocRaw(size, &allocated_size);
+		structPtr = ShmemAllocRawInSegment(size, &allocated_size, shmem_segment);
 		if (structPtr == NULL)
 		{
 			/* out of memory; remove the failed ShmemIndex entry */
@@ -478,14 +535,13 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
 
 	LWLockRelease(ShmemIndexLock);
 
-	Assert(ShmemAddrIsValid(structPtr));
+	Assert(ShmemAddrIsValidInSegment(structPtr, shmem_segment));
 
 	Assert(structPtr == (void *) CACHELINEALIGN(structPtr));
 
 	return structPtr;
 }
 
-
 /*
  * Add two Size values, checking for overflow
  */
@@ -542,10 +598,11 @@ pg_get_shmem_allocations(PG_FUNCTION_ARGS)
 
 	/* output all allocated entries */
 	memset(nulls, 0, sizeof(nulls));
+	/* XXX: take all shared memory segments into account. */
 	while ((ent = (ShmemIndexEnt *) hash_seq_search(&hstat)) != NULL)
 	{
 		values[0] = CStringGetTextDatum(ent->key);
-		values[1] = Int64GetDatum((char *) ent->location - (char *) ShmemSegHdr);
+		values[1] = Int64GetDatum((char *) ent->location - (char *) Segments[MAIN_SHMEM_SEGMENT].ShmemSegHdr);
 		values[2] = Int64GetDatum(ent->size);
 		values[3] = Int64GetDatum(ent->allocated_size);
 		named_allocated += ent->allocated_size;
@@ -557,15 +614,15 @@ pg_get_shmem_allocations(PG_FUNCTION_ARGS)
 	/* output shared memory allocated but not counted via the shmem index */
 	values[0] = CStringGetTextDatum("<anonymous>");
 	nulls[1] = true;
-	values[2] = Int64GetDatum(ShmemSegHdr->freeoffset - named_allocated);
+	values[2] = Int64GetDatum(Segments[MAIN_SHMEM_SEGMENT].ShmemSegHdr->freeoffset - named_allocated);
 	values[3] = values[2];
 	tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls);
 
 	/* output as-of-yet unused shared memory */
 	nulls[0] = true;
-	values[1] = Int64GetDatum(ShmemSegHdr->freeoffset);
+	values[1] = Int64GetDatum(Segments[MAIN_SHMEM_SEGMENT].ShmemSegHdr->freeoffset);
 	nulls[1] = false;
-	values[2] = Int64GetDatum(ShmemSegHdr->totalsize - ShmemSegHdr->freeoffset);
+	values[2] = Int64GetDatum(Segments[MAIN_SHMEM_SEGMENT].ShmemSegHdr->totalsize - Segments[MAIN_SHMEM_SEGMENT].ShmemSegHdr->freeoffset);
 	values[3] = values[2];
 	tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls);
 
@@ -630,7 +687,12 @@ pg_get_shmem_allocations_numa(PG_FUNCTION_ARGS)
 	 * this is not very likely, and moreover we have more entries, each of
 	 * them using only fraction of the total pages.
 	 */
-	shm_total_page_count = (ShmemSegHdr->totalsize / os_page_size) + 1;
+	for(int segment = 0; segment < ANON_MAPPINGS; segment++)
+	{
+		PGShmemHeader *shmhdr = Segments[segment].ShmemSegHdr;
+		shm_total_page_count += (shmhdr->totalsize / os_page_size) + 1;
+	}
+
 	page_ptrs = palloc0(sizeof(void *) * shm_total_page_count);
 	pages_status = palloc(sizeof(int) * shm_total_page_count);
 
diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c
index 46f44bc4511..a36b08895c8 100644
--- a/src/backend/storage/lmgr/lwlock.c
+++ b/src/backend/storage/lmgr/lwlock.c
@@ -80,6 +80,8 @@
 #include "pg_trace.h"
 #include "pgstat.h"
 #include "port/pg_bitutils.h"
+#include "postmaster/postmaster.h"
+#include "storage/pg_shmem.h"
 #include "storage/proc.h"
 #include "storage/proclist.h"
 #include "storage/procnumber.h"
@@ -618,10 +620,15 @@ LWLockNewTrancheId(void)
 	int		   *LWLockCounter;
 
 	LWLockCounter = (int *) ((char *) MainLWLockArray - sizeof(int));
-	/* We use the ShmemLock spinlock to protect LWLockCounter */
-	SpinLockAcquire(ShmemLock);
+	/*
+	 * We use the ShmemLock spinlock to protect LWLockCounter.
+	 *
+	 * XXX: Looks like this is the only use of Segments outside of shmem.c,
+	 * it's maybe worth it to reshape this part to hide Segments structure.
+	 */
+	SpinLockAcquire(Segments[MAIN_SHMEM_SEGMENT].ShmemLock);
 	result = (*LWLockCounter)++;
-	SpinLockRelease(ShmemLock);
+	SpinLockRelease(Segments[MAIN_SHMEM_SEGMENT].ShmemLock);
 
 	return result;
 }
diff --git a/src/include/storage/ipc.h b/src/include/storage/ipc.h
index 3baf418b3d1..6ebda479ced 100644
--- a/src/include/storage/ipc.h
+++ b/src/include/storage/ipc.h
@@ -77,7 +77,7 @@ extern void check_on_shmem_exit_lists_are_empty(void);
 /* ipci.c */
 extern PGDLLIMPORT shmem_startup_hook_type shmem_startup_hook;
 
-extern Size CalculateShmemSize(int *num_semaphores);
+extern Size CalculateShmemSize(int *num_semaphores, int shmem_segment);
 extern void CreateSharedMemoryAndSemaphores(void);
 #ifdef EXEC_BACKEND
 extern void AttachSharedMemoryStructs(void);
diff --git a/src/include/storage/pg_sema.h b/src/include/storage/pg_sema.h
index fa6ca35a51f..8ae9637fcd0 100644
--- a/src/include/storage/pg_sema.h
+++ b/src/include/storage/pg_sema.h
@@ -41,7 +41,7 @@ typedef HANDLE PGSemaphore;
 extern Size PGSemaphoreShmemSize(int maxSemas);
 
 /* Module initialization (called during postmaster start or shmem reinit) */
-extern void PGReserveSemaphores(int maxSemas);
+extern void PGReserveSemaphores(int maxSemas, int shmem_segment);
 
 /* Allocate a PGSemaphore structure with initial count 1 */
 extern PGSemaphore PGSemaphoreCreate(void);
diff --git a/src/include/storage/pg_shmem.h b/src/include/storage/pg_shmem.h
index 5f7d4b83a60..2348c59b5a0 100644
--- a/src/include/storage/pg_shmem.h
+++ b/src/include/storage/pg_shmem.h
@@ -25,6 +25,7 @@
 #define PG_SHMEM_H
 
 #include "storage/dsm_impl.h"
+#include "storage/spin.h"
 
 typedef struct PGShmemHeader	/* standard header for all Postgres shmem */
 {
@@ -41,6 +42,20 @@ typedef struct PGShmemHeader	/* standard header for all Postgres shmem */
 #endif
 } PGShmemHeader;
 
+typedef struct ShmemSegment
+{
+	PGShmemHeader *ShmemSegHdr; 	/* shared mem segment header */
+	void *ShmemBase; 				/* start address of shared memory */
+	void *ShmemEnd; 				/* end+1 address of shared memory */
+	slock_t    *ShmemLock; 			/* spinlock for shared memory and LWLock
+									 * allocation */
+} ShmemSegment;
+
+/* Number of available segments for anonymous memory mappings */
+#define ANON_MAPPINGS 1
+
+extern PGDLLIMPORT ShmemSegment Segments[ANON_MAPPINGS];
+
 /* GUC variables */
 extern PGDLLIMPORT int shared_memory_type;
 extern PGDLLIMPORT int huge_pages;
@@ -91,4 +106,7 @@ extern bool PGSharedMemoryIsInUse(unsigned long id1, unsigned long id2);
 extern void PGSharedMemoryDetach(void);
 extern void GetHugePageSize(Size *hugepagesize, int *mmap_flags);
 
+/* The main segment, contains everything except buffer blocks and related data. */
+#define MAIN_SHMEM_SEGMENT 0
+
 #endif							/* PG_SHMEM_H */
diff --git a/src/include/storage/shmem.h b/src/include/storage/shmem.h
index c1f668ded95..69663d412c3 100644
--- a/src/include/storage/shmem.h
+++ b/src/include/storage/shmem.h
@@ -29,15 +29,27 @@
 extern PGDLLIMPORT slock_t *ShmemLock;
 struct PGShmemHeader;			/* avoid including storage/pg_shmem.h here */
 extern void InitShmemAccess(struct PGShmemHeader *seghdr);
+extern void InitShmemAccessInSegment(struct PGShmemHeader *seghdr,
+									 int shmem_segment);
 extern void InitShmemAllocation(void);
+extern void InitShmemAllocationInSegment(int shmem_segment);
 extern void *ShmemAlloc(Size size);
+extern void *ShmemAllocInSegment(Size size, int shmem_segment);
 extern void *ShmemAllocNoError(Size size);
 extern void *ShmemAllocUnlocked(Size size);
+extern void *ShmemAllocUnlockedInSegment(Size size, int shmem_segment);
 extern bool ShmemAddrIsValid(const void *addr);
+extern bool ShmemAddrIsValidInSegment(const void *addr, int shmem_segment);
 extern void InitShmemIndex(void);
+extern void InitVariableShmemIndex(void);
 extern HTAB *ShmemInitHash(const char *name, long init_size, long max_size,
 						   HASHCTL *infoP, int hash_flags);
+extern HTAB *ShmemInitHashInSegment(const char *name, long init_size,
+									long max_size, HASHCTL *infoP,
+									int hash_flags, int shmem_segment);
 extern void *ShmemInitStruct(const char *name, Size size, bool *foundPtr);
+extern void *ShmemInitStructInSegment(const char *name, Size size,
+									  bool *foundPtr, int shmem_segment);
 extern Size add_size(Size s1, Size s2);
 extern Size mul_size(Size s1, Size s2);
 
-- 
2.49.0


--bwrlgp6w2ubxykjq
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
	filename="v5-0005-Address-space-reservation-for-shared-memory.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread

* [PATCH v3 1/2] ci: Improve ccache handling
@ 2026-06-05 15:58  Andres Freund <[email protected]>
  0 siblings, 0 replies; 436+ messages in thread

From: Andres Freund @ 2026-06-05 15:58 UTC (permalink / raw)

There previously were a number of issues:

- We'd upload the cache even if we already had a high hit rate. That means we
  churn through the available cache space very quickly.

  For this we now check if the cache hit ratio is already high, and skip
  uploading a new cache in that case.

- We'd generate per-branch caches, even if master's already would suffice,
  because the branch doesn't change much

  This is solved indirectly by the above.

- The cache key allowed prefix matches based on the branch,
  e.g. master-pending would always use master's branch

  Replace the cache key element separator of - with :, which is not a valid
  part of a branch name.

- When rebasing a feature branch, we'd start with just that branch's cache,
  rather than also having the newer cache of master available

  This is solved by downloading by master's and the feature branch's cache,
  simply overlaying both. That's possible because ccache is content addressed.

- The size of a cache would increase to the max, even though there likely will
  be no benefit from old cache entries.

  Address this by explicitly evicting old data and also recompressing the
  cache before uploading it.

In my testing this utilizes the available cache space (10GB for personal
accounts) much more effictively than before.

The not entirely trivial determination of whether it's worth uploading a cache
entry is moved to a python script.  I first had it as shell, but that gets
awkward.  This way it'd also be more viable to use ccache for msvc at some
point.

The per-job redundancies are a bit annoying. There's a way around that, by
using composite actions, but I think that might be harder to understand,
without all that much of an improvement.
---
 .github/workflows/pg-ci.yml       |  94 ++++++++++++++++++++++------
 src/tools/ci/gha_ccache_decide.py | 100 ++++++++++++++++++++++++++++++
 2 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 src/tools/ci/gha_ccache_decide.py

diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml
index 8560e9389f6..86dc47de8db 100644
--- a/.github/workflows/pg-ci.yml
+++ b/.github/workflows/pg-ci.yml
@@ -130,6 +130,22 @@ env:
   # commit-message directive parsed in the `setup` job below.
   CI_OS_ONLY_JOBS: "linux macos windows mingw compilerwarnings sanitycheck"
 
+  ###
+  # A few variables to make expressions later on shorter
+  ###
+
+  ON_DEFAULT_BRANCH: ${{github.event.repository.default_branch == github.ref_name }}
+
+  # Note that we need to be careful to use a separator that can't be in branch
+  # names, otherwise e.g. caches for 'master' might be restored on the
+  # 'master-pending' branch.
+  CACHE_PREFIX_DEFAULT: >-
+    :${{ github.job }}:${{ github.event.repository.default_branch }}:
+  CACHE_PREFIX_BRANCH: >-
+    :${{ github.job }}:${{ github.ref_name }}:
+  CACHE_SUFFIX: >-
+    ${{ github.run_id }}:${{ github.run_attempt }}
+
 
 jobs:
 
@@ -277,16 +293,30 @@ jobs:
         with:
           fetch-depth: ${{ env.CLONE_DEPTH }}
 
-      - &ccache_restore_step
-        name: Restore ccache
-        id: ccache_restore
+      # We restore both the ccache from the default branch (typically master),
+      # and from the current branch. This will often allow feature branches to
+      # start out with a high cache hit ratio.
+      #
+      # With ccache it turns out to work to just restore two caches into the
+      # same directory, as it's basically a content addressed store. Stats
+      # could be corrupted, but we zero them out anyway.
+      - &ccache_restore_default_step
+        name: "ccache: Restore for default branch ${{github.event.repository.default_branch}}"
+        if: ${{ env.ON_DEFAULT_BRANCH == 'false' }}
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ccache-${{ github.job }}-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
-          restore-keys: |
-            ccache-${{ github.job }}-${{ github.ref_name }}-
-            ccache-${{ github.job }}-
+          key: ccache${{env.CACHE_PREFIX_DEFAULT}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_DEFAULT}}
+
+      - &ccache_restore_branch_step
+        name: "ccache: Restore for branch ${{ github.ref_name }}"
+        id: ccache-restore-branch
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache${{env.CACHE_PREFIX_BRANCH}}${{env.CACHE_SUFFIX}}
+          restore-keys: ccache${{env.CACHE_PREFIX_BRANCH}}
 
       - &linux_prepare_workspace_step
         name: Prepare workspace
@@ -325,15 +355,30 @@ jobs:
           ninja -C build -j${{env.BUILD_JOBS}} ${{env.MBUILD_TARGET}}
           ninja -C build -t missingdeps
 
-      # TODO: As long as we use per-run ccache caches, we should probably add
-      # a step that checks if there is sufficient new content to warrant
-      # saving the new cache.
+      # Decide if it's worth uploading a new version of the ccache cache. If
+      # we always do so unconditionally, we'd very quickly go through the
+      # allowed cache space. Instead we check if the hit rate is high enough
+      # already for that not to be worth it.
+      - &ccache_decide_save_step
+        name: "ccache: Decide if cache should be uploaded"
+        id: ccache-pre-save
+        # [Decide to] store the cache whenever the cache was set up, so that
+        # incrementally addressing compiler errors/warnings doesn't have to
+        # start from scratch.
+        if: |
+          always() &&
+          steps.ccache-restore-branch.conclusion == 'success'
+        run: python3 src/tools/ci/gha_ccache_decide.py
+
       - &ccache_save_step
-        name: Save ccache
+        name: "ccache: Upload cache"
         uses: actions/cache/save@v5
+        if: |
+          always() &&
+          steps.ccache-pre-save.outputs.should_save == 'true'
         with:
           path: ${{ env.CCACHE_DIR }}
-          key: ${{ steps.ccache_restore.outputs.cache-primary-key }}
+          key: ${{ steps.ccache-restore-branch.outputs.cache-primary-key }}
 
       # Run a minimal set of tests. The main regression tests take too long
       # for this purpose. For now this is a random quick pg_regress style
@@ -448,7 +493,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -467,6 +513,7 @@ jobs:
         run: |
           make -s -j${BUILD_JOBS} world-bin
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -508,7 +555,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -527,6 +575,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -596,7 +645,8 @@ jobs:
 
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
       - *linux_prepare_workspace_step
 
       - name: Configure
@@ -613,6 +663,7 @@ jobs:
         shell: *su_postgres_shell
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -682,7 +733,6 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
 
       - name: Setup core files
         run: |
@@ -745,6 +795,9 @@ jobs:
           path: ${{ env.MACPORTS_CACHE }}
           key: ${{ steps.mp-key.outputs.key }}
 
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
+
       - name: Configure
         env:
           PKG_CONFIG_PATH: /opt/local/lib/pkgconfig/
@@ -762,6 +815,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1062,7 +1116,8 @@ jobs:
         shell: cmd
         run: mkdir ${{env.PG_REGRESS_SOCK_DIR}}
 
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Configure
         run: |
@@ -1077,6 +1132,7 @@ jobs:
       - name: Build
         run: *ninja_build_cmd
 
+      - *ccache_decide_save_step
       - *ccache_save_step
 
       - name: Test world
@@ -1118,7 +1174,8 @@ jobs:
     steps:
       - *nix_sysinfo_step
       - *checkout_step
-      - *ccache_restore_step
+      - *ccache_restore_default_step
+      - *ccache_restore_branch_step
 
       - name: Setup workspace
         run: |
@@ -1213,5 +1270,6 @@ jobs:
             headerscheck cpluspluscheck \
             EXTRAFLAGS='-fmax-errors=10'
 
+      - *ccache_decide_save_step
       - *ccache_save_step
       - *upload_logs_step
diff --git a/src/tools/ci/gha_ccache_decide.py b/src/tools/ci/gha_ccache_decide.py
new file mode 100644
index 00000000000..920f7bf9685
--- /dev/null
+++ b/src/tools/ci/gha_ccache_decide.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shutil
+import subprocess
+
+def run(cmd, check=True):
+    return subprocess.run(
+        cmd,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    ).stdout
+
+def parse_ccache_stats():
+    out = run(["ccache", "--print-stats"])
+    hits = 0
+    misses = 0
+
+    for line in out.splitlines():
+        line = line.strip()
+        m = re.match(r"^local_storage_hit\s+(\d+)$", line)
+        if m:
+            hits = int(m.group(1))
+            continue
+        m = re.match(r"^local_storage_miss\s+(\d+)$", line)
+        if m:
+            misses = int(m.group(1))
+            continue
+
+    return hits, misses
+
+def append_github_output(key, value):
+    output_path = os.environ["GITHUB_OUTPUT"]
+    with open(output_path, "a", encoding="utf-8") as f:
+        f.write(f"{key}={value}\n")
+
+def main():
+    on_default_branch = os.environ["ON_DEFAULT_BRANCH"] == "true"
+    ccache_dir = os.environ["CCACHE_DIR"]
+
+    # Decide the target hit percentage below which we decide to upload a new
+    # cache. On non-default branches a few misses aren't that bad. But, as the
+    # caches of the default branch are shared with all branches, it's worth
+    # aiming for a higher ratio there.
+    target_rate = 95 if on_default_branch else 80
+
+    # Log ccache stats, useful for more in-depth understanding. The avoid it
+    # swamping the output, collapse it in a group.
+    print("::group::ccache_stats")
+    print(run(["ccache", "-s", "-vv"]))
+    print("::endgroup::")
+
+    # compute cache hit ratio
+    hits, misses = parse_ccache_stats()
+    total = hits + misses
+    hit_pct = int(( hits / total) * 100) if total > 0 else 100
+
+    print(f"hits: {hits}, misses: {misses}, hit_pct: {hit_pct}, target rate: {target_rate}")
+
+    # If there were either barely any misses, or the cache hit ratio was high,
+    # there no point in generating a new cache entry. We have limited cache
+    # space.
+    should_save = misses > 10 and hit_pct < target_rate
+
+    append_github_output("should_save", str(should_save).lower())
+
+    if not should_save:
+        print(f"hit rate {hit_pct} is above target of {target_rate}, skip creating new cache entry")
+        return 0
+
+    print(f"hit rate {hit_pct} is below target of {target_rate}, create new cache entry")
+
+    # It's not worth persisting old cache entries (e.g. from before a
+    # change to a central header, or from the default branch if this
+    # branch differs a lot). Therefore evict ccache entries that are a
+    # bit older. The cutoff here is fairly arbitrary, it could
+    # probably be improved.
+    print("::group::ccache_shrink")
+    print(run(["ccache", "--evict-older-than", f"{45*60}s"]))
+    print(run(["ccache", "-X", "10"]))
+
+    # Don't store ccache stats , otherwise we'd need to reset the cache access
+    # data after restoring the cache in the next run, to be able to get the
+    # hit ratio of the CI run.
+    print(run(["ccache", "-z"]))
+    print("::endgroup::")
+
+    # Before continuing, try to kill all ccache instances, otherwise
+    # it's possible that on cancellations there is still running
+    # ccaches that cause the upload to fail.
+    if shutil.which("killall"):
+        print(run(["killall", "ccache"], check=False))
+
+    return 0
+
+if __name__ == "__main__":
+    exit(main())
-- 
2.54.0.380.gc69baaf57b


--zzyr2ozda5zjkkci
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v3-0002-ci-fewer-tests.patch"



^ permalink  raw  reply  [nested|flat] 436+ messages in thread