agora inbox for pgsql-hackers@postgresql.org
help / color / mirror / Atom feed[PATCH v1 1/5] Allow to use multiple shared memory mappings
9+ messages / 2 participants
[nested] [flat]
* [PATCH v1 1/5] Allow to use multiple shared memory mappings
@ 2024-10-09 13:41 Dmitrii Dolgov <9erthalion6@gmail.com>
0 siblings, 0 replies; 9+ messages in thread
From: Dmitrii Dolgov @ 2024-10-09 13:41 UTC (permalink / raw)
Currently all the work with shared memory is done via a single anonymous
memory mapping, which limits ways how the shared memory could be organized.
Introduce possibility to allocate multiple shared memory mappings, where
a single mapping is associated with a specified shared memory slot.
There is only fixed amount of available slots, currently only one main
shared memory slot is allocated. A new shared memory API is introduces,
extended with a slot as a new parameter. As a path of least resistance,
the original API is kept in place, utilizing the main shared memory slot.
---
src/backend/port/posix_sema.c | 4 +-
src/backend/port/sysv_sema.c | 4 +-
src/backend/port/sysv_shmem.c | 138 +++++++++++++++++++---------
src/backend/port/win32_sema.c | 2 +-
src/backend/storage/ipc/ipc.c | 2 +-
src/backend/storage/ipc/ipci.c | 61 ++++++------
src/backend/storage/ipc/shmem.c | 133 ++++++++++++++++++---------
src/backend/storage/lmgr/lwlock.c | 5 +-
src/include/storage/buf_internals.h | 1 +
src/include/storage/ipc.h | 2 +-
src/include/storage/pg_sema.h | 2 +-
src/include/storage/pg_shmem.h | 18 ++++
src/include/storage/shmem.h | 10 ++
13 files changed, 258 insertions(+), 124 deletions(-)
diff --git a/src/backend/port/posix_sema.c b/src/backend/port/posix_sema.c
index 64186ec0a7..b97723d2ed 100644
--- a/src/backend/port/posix_sema.c
+++ b/src/backend/port/posix_sema.c
@@ -193,7 +193,7 @@ PGSemaphoreShmemSize(int maxSemas)
* we don't have to expose the counters to other processes.)
*/
void
-PGReserveSemaphores(int maxSemas)
+PGReserveSemaphores(int maxSemas, int shmem_slot)
{
struct stat statbuf;
@@ -220,7 +220,7 @@ PGReserveSemaphores(int maxSemas)
* ShmemAlloc() won't be ready yet.
*/
sharedSemas = (PGSemaphore)
- ShmemAllocUnlocked(PGSemaphoreShmemSize(maxSemas));
+ ShmemAllocUnlockedInSlot(PGSemaphoreShmemSize(maxSemas), shmem_slot);
#endif
numSems = 0;
diff --git a/src/backend/port/sysv_sema.c b/src/backend/port/sysv_sema.c
index 5b88a92bc9..8ef95b12c9 100644
--- a/src/backend/port/sysv_sema.c
+++ b/src/backend/port/sysv_sema.c
@@ -307,7 +307,7 @@ PGSemaphoreShmemSize(int maxSemas)
* have clobbered.)
*/
void
-PGReserveSemaphores(int maxSemas)
+PGReserveSemaphores(int maxSemas, int shmem_slot)
{
struct stat statbuf;
@@ -328,7 +328,7 @@ PGReserveSemaphores(int maxSemas)
* ShmemAlloc() won't be ready yet.
*/
sharedSemas = (PGSemaphore)
- ShmemAllocUnlocked(PGSemaphoreShmemSize(maxSemas));
+ ShmemAllocUnlockedInSlot(PGSemaphoreShmemSize(maxSemas), shmem_slot);
numSharedSemas = 0;
maxSharedSemas = maxSemas;
diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 362a37d3b3..065a5b63ac 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -94,8 +94,19 @@ typedef enum
unsigned long UsedShmemSegID = 0;
void *UsedShmemSegAddr = NULL;
-static Size AnonymousShmemSize;
-static void *AnonymousShmem = NULL;
+typedef struct AnonymousMapping
+{
+ int shmem_slot;
+ Size shmem_size; /* Size of the mapping */
+ void *shmem; /* Pointer to the start of the mapped memory */
+ void *seg_addr; /* SysV shared memory for the header */
+ unsigned long seg_id; /* IPC key */
+} AnonymousMapping;
+
+static AnonymousMapping Mappings[ANON_MAPPINGS];
+
+/* Keeps track of used mapping slots */
+static int next_free_slot = 0;
static void *InternalIpcMemoryCreate(IpcMemoryKey memKey, Size size);
static void IpcMemoryDetach(int status, Datum shmaddr);
@@ -104,6 +115,28 @@ static IpcMemoryState PGSharedMemoryAttach(IpcMemoryId shmId,
void *attachAt,
PGShmemHeader **addr);
+static const char*
+MappingName(int shmem_slot)
+{
+ switch (shmem_slot)
+ {
+ case MAIN_SHMEM_SLOT:
+ return "main";
+ default:
+ return "unknown";
+ }
+}
+
+static void
+DebugMappings()
+{
+ for(int i = 0; i < next_free_slot; i++)
+ {
+ AnonymousMapping m = Mappings[i];
+ elog(DEBUG1, "Mapping[%s]: addr %p, size %zu",
+ MappingName(i), m.shmem, m.shmem_size);
+ }
+}
/*
* InternalIpcMemoryCreate(memKey, size)
@@ -591,14 +624,13 @@ check_huge_page_size(int *newval, void **extra, GucSource source)
/*
* Creates an anonymous mmap()ed shared memory segment.
*
- * Pass the requested size in *size. This function will modify *size to the
- * actual size of the allocation, if it ends up allocating a segment that is
- * larger than requested.
+ * This function will modify mapping size to the actual size of the allocation,
+ * if it ends up allocating a segment that is larger than requested.
*/
-static void *
-CreateAnonymousSegment(Size *size)
+static void
+CreateAnonymousSegment(AnonymousMapping *mapping)
{
- Size allocsize = *size;
+ Size allocsize = mapping->shmem_size;
void *ptr = MAP_FAILED;
int mmap_errno = 0;
@@ -623,8 +655,11 @@ CreateAnonymousSegment(Size *size)
PG_MMAP_FLAGS | mmap_flags, -1, 0);
mmap_errno = errno;
if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
- elog(DEBUG1, "mmap(%zu) with MAP_HUGETLB failed, huge pages disabled: %m",
- allocsize);
+ {
+ DebugMappings();
+ elog(DEBUG1, "slot[%s]: mmap(%zu) with MAP_HUGETLB failed, huge pages disabled: %m",
+ MappingName(mapping->shmem_slot), allocsize);
+ }
}
#endif
@@ -642,7 +677,7 @@ CreateAnonymousSegment(Size *size)
* Use the original size, not the rounded-up value, when falling back
* to non-huge pages.
*/
- allocsize = *size;
+ allocsize = mapping->shmem_size;
ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
PG_MMAP_FLAGS, -1, 0);
mmap_errno = errno;
@@ -651,8 +686,10 @@ CreateAnonymousSegment(Size *size)
if (ptr == MAP_FAILED)
{
errno = mmap_errno;
+ DebugMappings();
ereport(FATAL,
- (errmsg("could not map anonymous shared memory: %m"),
+ (errmsg("slot[%s]: could not map anonymous shared memory: %m",
+ MappingName(mapping->shmem_slot)),
(mmap_errno == ENOMEM) ?
errhint("This error usually means that PostgreSQL's request "
"for a shared memory segment exceeded available memory, "
@@ -663,8 +700,8 @@ CreateAnonymousSegment(Size *size)
allocsize) : 0));
}
- *size = allocsize;
- return ptr;
+ mapping->shmem = ptr;
+ mapping->shmem_size = allocsize;
}
/*
@@ -674,13 +711,18 @@ CreateAnonymousSegment(Size *size)
static void
AnonymousShmemDetach(int status, Datum arg)
{
- /* Release anonymous shared memory block, if any. */
- if (AnonymousShmem != NULL)
+ for(int i = 0; i < next_free_slot; i++)
{
- if (munmap(AnonymousShmem, AnonymousShmemSize) < 0)
- elog(LOG, "munmap(%p, %zu) failed: %m",
- AnonymousShmem, AnonymousShmemSize);
- AnonymousShmem = NULL;
+ AnonymousMapping m = Mappings[i];
+
+ /* Release anonymous shared memory block, if any. */
+ if (m.shmem != NULL)
+ {
+ if (munmap(m.shmem, m.shmem_size) < 0)
+ elog(LOG, "munmap(%p, %zu) failed: %m",
+ m.shmem, m.shmem_size);
+ m.shmem = NULL;
+ }
}
}
@@ -705,6 +747,7 @@ PGSharedMemoryCreate(Size size,
PGShmemHeader *hdr;
struct stat statbuf;
Size sysvsize;
+ AnonymousMapping *mapping = &Mappings[next_free_slot];
/*
* We use the data directory's ID info (inode and device numbers) to
@@ -733,11 +776,15 @@ PGSharedMemoryCreate(Size size,
/* Room for a header? */
Assert(size > MAXALIGN(sizeof(PGShmemHeader)));
+ mapping->shmem_size = size;
+ mapping->shmem_slot = next_free_slot;
if (shared_memory_type == SHMEM_TYPE_MMAP)
{
- AnonymousShmem = CreateAnonymousSegment(&size);
- AnonymousShmemSize = size;
+ /* On success, mapping data will be modified. */
+ CreateAnonymousSegment(mapping);
+
+ next_free_slot++;
/* Register on-exit routine to unmap the anonymous segment */
on_shmem_exit(AnonymousShmemDetach, (Datum) 0);
@@ -760,7 +807,7 @@ PGSharedMemoryCreate(Size size,
* loop simultaneously. (CreateDataDirLockFile() does not entirely ensure
* that, but prefer fixing it over coping here.)
*/
- NextShmemSegID = statbuf.st_ino;
+ NextShmemSegID = statbuf.st_ino + next_free_slot;
for (;;)
{
@@ -852,13 +899,13 @@ PGSharedMemoryCreate(Size size,
/*
* Initialize space allocation status for segment.
*/
- hdr->totalsize = size;
+ hdr->totalsize = mapping->shmem_size;
hdr->freeoffset = MAXALIGN(sizeof(PGShmemHeader));
*shim = hdr;
/* Save info for possible future use */
- UsedShmemSegAddr = memAddress;
- UsedShmemSegID = (unsigned long) NextShmemSegID;
+ mapping->seg_addr = memAddress;
+ mapping->seg_id = (unsigned long) NextShmemSegID;
/*
* If AnonymousShmem is NULL here, then we're not using anonymous shared
@@ -866,10 +913,10 @@ PGSharedMemoryCreate(Size size,
* block. Otherwise, the System V shared memory block is only a shim, and
* we must return a pointer to the real block.
*/
- if (AnonymousShmem == NULL)
+ if (mapping->shmem == NULL)
return hdr;
- memcpy(AnonymousShmem, hdr, sizeof(PGShmemHeader));
- return (PGShmemHeader *) AnonymousShmem;
+ memcpy(mapping->shmem, hdr, sizeof(PGShmemHeader));
+ return (PGShmemHeader *) mapping->shmem;
}
#ifdef EXEC_BACKEND
@@ -969,23 +1016,28 @@ PGSharedMemoryNoReAttach(void)
void
PGSharedMemoryDetach(void)
{
- if (UsedShmemSegAddr != NULL)
+ for(int i = 0; i < next_free_slot; i++)
{
- if ((shmdt(UsedShmemSegAddr) < 0)
+ AnonymousMapping m = Mappings[i];
+
+ if (m.seg_addr != NULL)
+ {
+ if ((shmdt(m.seg_addr) < 0)
#if defined(EXEC_BACKEND) && defined(__CYGWIN__)
- /* Work-around for cygipc exec bug */
- && shmdt(NULL) < 0
+ /* Work-around for cygipc exec bug */
+ && shmdt(NULL) < 0
#endif
- )
- elog(LOG, "shmdt(%p) failed: %m", UsedShmemSegAddr);
- UsedShmemSegAddr = NULL;
- }
+ )
+ elog(LOG, "shmdt(%p) failed: %m", m.seg_addr);
+ m.seg_addr = NULL;
+ }
- if (AnonymousShmem != NULL)
- {
- if (munmap(AnonymousShmem, AnonymousShmemSize) < 0)
- elog(LOG, "munmap(%p, %zu) failed: %m",
- AnonymousShmem, AnonymousShmemSize);
- AnonymousShmem = NULL;
+ if (m.shmem != NULL)
+ {
+ if (munmap(m.shmem, m.shmem_size) < 0)
+ elog(LOG, "munmap(%p, %zu) failed: %m",
+ m.shmem, m.shmem_size);
+ m.shmem = NULL;
+ }
}
}
diff --git a/src/backend/port/win32_sema.c b/src/backend/port/win32_sema.c
index f2b54bdfda..d62084cc0d 100644
--- a/src/backend/port/win32_sema.c
+++ b/src/backend/port/win32_sema.c
@@ -44,7 +44,7 @@ PGSemaphoreShmemSize(int maxSemas)
* process exits.
*/
void
-PGReserveSemaphores(int maxSemas)
+PGReserveSemaphores(int maxSemas, int shmem_slot)
{
mySemSet = (HANDLE *) malloc(maxSemas * sizeof(HANDLE));
if (mySemSet == NULL)
diff --git a/src/backend/storage/ipc/ipc.c b/src/backend/storage/ipc/ipc.c
index b06e4b8452..2aabd4a77f 100644
--- a/src/backend/storage/ipc/ipc.c
+++ b/src/backend/storage/ipc/ipc.c
@@ -68,7 +68,7 @@ static void proc_exit_prepare(int code);
* ----------------------------------------------------------------
*/
-#define MAX_ON_EXITS 20
+#define MAX_ON_EXITS 40
struct ONEXIT
{
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 35fa2e1dda..8224015b53 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -88,7 +88,7 @@ RequestAddinShmemSpace(Size size)
* required.
*/
Size
-CalculateShmemSize(int *num_semaphores)
+CalculateShmemSize(int *num_semaphores, int shmem_slot)
{
Size size;
int numSemas;
@@ -202,33 +202,36 @@ CreateSharedMemoryAndSemaphores(void)
Assert(!IsUnderPostmaster);
- /* Compute the size of the shared-memory block */
- size = CalculateShmemSize(&numSemas);
- elog(DEBUG3, "invoking IpcMemoryCreate(size=%zu)", size);
-
- /*
- * Create the shmem segment
- */
- seghdr = PGSharedMemoryCreate(size, &shim);
-
- /*
- * Make sure that huge pages are never reported as "unknown" while the
- * server is running.
- */
- Assert(strcmp("unknown",
- GetConfigOption("huge_pages_status", false, false)) != 0);
-
- InitShmemAccess(seghdr);
-
- /*
- * Create semaphores
- */
- PGReserveSemaphores(numSemas);
-
- /*
- * Set up shared memory allocation mechanism
- */
- InitShmemAllocation();
+ for(int slot = 0; slot < ANON_MAPPINGS; slot++)
+ {
+ /* Compute the size of the shared-memory block */
+ size = CalculateShmemSize(&numSemas, slot);
+ elog(DEBUG3, "invoking IpcMemoryCreate(size=%zu)", size);
+
+ /*
+ * Create the shmem segment
+ */
+ seghdr = PGSharedMemoryCreate(size, &shim);
+
+ /*
+ * Make sure that huge pages are never reported as "unknown" while the
+ * server is running.
+ */
+ Assert(strcmp("unknown",
+ GetConfigOption("huge_pages_status", false, false)) != 0);
+
+ InitShmemAccessInSlot(seghdr, slot);
+
+ /*
+ * Create semaphores
+ */
+ PGReserveSemaphores(numSemas, slot);
+
+ /*
+ * Set up shared memory allocation mechanism
+ */
+ InitShmemAllocationInSlot(slot);
+ }
/* Initialize subsystems */
CreateOrAttachShmemStructs();
@@ -359,7 +362,7 @@ InitializeShmemGUCs(void)
/*
* Calculate the shared memory size and round up to the nearest megabyte.
*/
- size_b = CalculateShmemSize(&num_semas);
+ size_b = CalculateShmemSize(&num_semas, MAIN_SHMEM_SLOT);
size_mb = add_size(size_b, (1024 * 1024) - 1) / (1024 * 1024);
sprintf(buf, "%zu", size_mb);
SetConfigOption("shared_memory_size", buf,
diff --git a/src/backend/storage/ipc/shmem.c b/src/backend/storage/ipc/shmem.c
index 6d5f083986..c670b9cf43 100644
--- a/src/backend/storage/ipc/shmem.c
+++ b/src/backend/storage/ipc/shmem.c
@@ -75,17 +75,12 @@
#include "utils/builtins.h"
static void *ShmemAllocRaw(Size size, Size *allocated_size);
+static void *ShmemAllocRawInSlot(Size size, Size *allocated_size,
+ int shmem_slot);
/* shared memory global variables */
-static PGShmemHeader *ShmemSegHdr; /* shared mem segment header */
-
-static void *ShmemBase; /* start address of shared memory */
-
-static void *ShmemEnd; /* end+1 address of shared memory */
-
-slock_t *ShmemLock; /* spinlock for shared memory and LWLock
- * allocation */
+ShmemSegment Segments[ANON_MAPPINGS];
static HTAB *ShmemIndex = NULL; /* primary index hashtable for shmem */
@@ -99,11 +94,17 @@ static HTAB *ShmemIndex = NULL; /* primary index hashtable for shmem */
void
InitShmemAccess(void *seghdr)
{
- PGShmemHeader *shmhdr = (PGShmemHeader *) seghdr;
+ InitShmemAccessInSlot(seghdr, MAIN_SHMEM_SLOT);
+}
- ShmemSegHdr = shmhdr;
- ShmemBase = (void *) shmhdr;
- ShmemEnd = (char *) ShmemBase + shmhdr->totalsize;
+void
+InitShmemAccessInSlot(void *seghdr, int shmem_slot)
+{
+ PGShmemHeader *shmhdr = (PGShmemHeader *) seghdr;
+ ShmemSegment *seg = &Segments[shmem_slot];
+ seg->ShmemSegHdr = shmhdr;
+ seg->ShmemBase = (void *) shmhdr;
+ seg->ShmemEnd = (char *) seg->ShmemBase + shmhdr->totalsize;
}
/*
@@ -114,7 +115,13 @@ InitShmemAccess(void *seghdr)
void
InitShmemAllocation(void)
{
- PGShmemHeader *shmhdr = ShmemSegHdr;
+ InitShmemAllocationInSlot(MAIN_SHMEM_SLOT);
+}
+
+void
+InitShmemAllocationInSlot(int shmem_slot)
+{
+ PGShmemHeader *shmhdr = Segments[shmem_slot].ShmemSegHdr;
char *aligned;
Assert(shmhdr != NULL);
@@ -123,9 +130,9 @@ InitShmemAllocation(void)
* Initialize the spinlock used by ShmemAlloc. We must use
* ShmemAllocUnlocked, since obviously ShmemAlloc can't be called yet.
*/
- ShmemLock = (slock_t *) ShmemAllocUnlocked(sizeof(slock_t));
+ Segments[shmem_slot].ShmemLock = (slock_t *) ShmemAllocUnlockedInSlot(sizeof(slock_t), shmem_slot);
- SpinLockInit(ShmemLock);
+ SpinLockInit(Segments[shmem_slot].ShmemLock);
/*
* Allocations after this point should go through ShmemAlloc, which
@@ -150,11 +157,17 @@ InitShmemAllocation(void)
*/
void *
ShmemAlloc(Size size)
+{
+ return ShmemAllocInSlot(size, MAIN_SHMEM_SLOT);
+}
+
+void *
+ShmemAllocInSlot(Size size, int shmem_slot)
{
void *newSpace;
Size allocated_size;
- newSpace = ShmemAllocRaw(size, &allocated_size);
+ newSpace = ShmemAllocRawInSlot(size, &allocated_size, shmem_slot);
if (!newSpace)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -184,6 +197,12 @@ ShmemAllocNoError(Size size)
*/
static void *
ShmemAllocRaw(Size size, Size *allocated_size)
+{
+ return ShmemAllocRawInSlot(size, allocated_size, MAIN_SHMEM_SLOT);
+}
+
+static void *
+ShmemAllocRawInSlot(Size size, Size *allocated_size, int shmem_slot)
{
Size newStart;
Size newFree;
@@ -203,22 +222,22 @@ ShmemAllocRaw(Size size, Size *allocated_size)
size = CACHELINEALIGN(size);
*allocated_size = size;
- Assert(ShmemSegHdr != NULL);
+ Assert(Segments[shmem_slot].ShmemSegHdr != NULL);
- SpinLockAcquire(ShmemLock);
+ SpinLockAcquire(Segments[shmem_slot].ShmemLock);
- newStart = ShmemSegHdr->freeoffset;
+ newStart = Segments[shmem_slot].ShmemSegHdr->freeoffset;
newFree = newStart + size;
- if (newFree <= ShmemSegHdr->totalsize)
+ if (newFree <= Segments[shmem_slot].ShmemSegHdr->totalsize)
{
- newSpace = (void *) ((char *) ShmemBase + newStart);
- ShmemSegHdr->freeoffset = newFree;
+ newSpace = (void *) ((char *) Segments[shmem_slot].ShmemBase + newStart);
+ Segments[shmem_slot].ShmemSegHdr->freeoffset = newFree;
}
else
newSpace = NULL;
- SpinLockRelease(ShmemLock);
+ SpinLockRelease(Segments[shmem_slot].ShmemLock);
/* note this assert is okay with newSpace == NULL */
Assert(newSpace == (void *) CACHELINEALIGN(newSpace));
@@ -236,6 +255,12 @@ ShmemAllocRaw(Size size, Size *allocated_size)
*/
void *
ShmemAllocUnlocked(Size size)
+{
+ return ShmemAllocUnlockedInSlot(size, MAIN_SHMEM_SLOT);
+}
+
+void *
+ShmemAllocUnlockedInSlot(Size size, int shmem_slot)
{
Size newStart;
Size newFree;
@@ -246,19 +271,19 @@ ShmemAllocUnlocked(Size size)
*/
size = MAXALIGN(size);
- Assert(ShmemSegHdr != NULL);
+ Assert(Segments[shmem_slot].ShmemSegHdr != NULL);
- newStart = ShmemSegHdr->freeoffset;
+ newStart = Segments[shmem_slot].ShmemSegHdr->freeoffset;
newFree = newStart + size;
- if (newFree > ShmemSegHdr->totalsize)
+ if (newFree > Segments[shmem_slot].ShmemSegHdr->totalsize)
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of shared memory (%zu bytes requested)",
size)));
- ShmemSegHdr->freeoffset = newFree;
+ Segments[shmem_slot].ShmemSegHdr->freeoffset = newFree;
- newSpace = (void *) ((char *) ShmemBase + newStart);
+ newSpace = (void *) ((char *) Segments[shmem_slot].ShmemBase + newStart);
Assert(newSpace == (void *) MAXALIGN(newSpace));
@@ -273,7 +298,13 @@ ShmemAllocUnlocked(Size size)
bool
ShmemAddrIsValid(const void *addr)
{
- return (addr >= ShmemBase) && (addr < ShmemEnd);
+ return ShmemAddrIsValidInSlot(addr, MAIN_SHMEM_SLOT);
+}
+
+bool
+ShmemAddrIsValidInSlot(const void *addr, int shmem_slot)
+{
+ return (addr >= Segments[shmem_slot].ShmemBase) && (addr < Segments[shmem_slot].ShmemEnd);
}
/*
@@ -334,6 +365,18 @@ ShmemInitHash(const char *name, /* table string name for shmem index */
long max_size, /* max size of the table */
HASHCTL *infoP, /* info about key and bucket size */
int hash_flags) /* info about infoP */
+{
+ return ShmemInitHashInSlot(name, init_size, max_size, infoP, hash_flags,
+ MAIN_SHMEM_SLOT);
+}
+
+HTAB *
+ShmemInitHashInSlot(const char *name, /* table string name for shmem index */
+ long init_size, /* initial table size */
+ long max_size, /* max size of the table */
+ HASHCTL *infoP, /* info about key and bucket size */
+ int hash_flags, /* info about infoP */
+ int shmem_slot) /* in which slot to keep the table */
{
bool found;
void *location;
@@ -350,9 +393,9 @@ ShmemInitHash(const char *name, /* table string name for shmem index */
hash_flags |= HASH_SHARED_MEM | HASH_ALLOC | HASH_DIRSIZE;
/* look it up in the shmem index */
- location = ShmemInitStruct(name,
+ location = ShmemInitStructInSlot(name,
hash_get_shared_size(infoP, hash_flags),
- &found);
+ &found, shmem_slot);
/*
* if it already exists, attach to it rather than allocate and initialize
@@ -385,6 +428,13 @@ ShmemInitHash(const char *name, /* table string name for shmem index */
*/
void *
ShmemInitStruct(const char *name, Size size, bool *foundPtr)
+{
+ return ShmemInitStructInSlot(name, size, foundPtr, MAIN_SHMEM_SLOT);
+}
+
+void *
+ShmemInitStructInSlot(const char *name, Size size, bool *foundPtr,
+ int shmem_slot)
{
ShmemIndexEnt *result;
void *structPtr;
@@ -393,7 +443,7 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
if (!ShmemIndex)
{
- PGShmemHeader *shmemseghdr = ShmemSegHdr;
+ PGShmemHeader *shmemseghdr = Segments[shmem_slot].ShmemSegHdr;
/* Must be trying to create/attach to ShmemIndex itself */
Assert(strcmp(name, "ShmemIndex") == 0);
@@ -416,7 +466,7 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
* process can be accessing shared memory yet.
*/
Assert(shmemseghdr->index == NULL);
- structPtr = ShmemAlloc(size);
+ structPtr = ShmemAllocInSlot(size, shmem_slot);
shmemseghdr->index = structPtr;
*foundPtr = false;
}
@@ -433,8 +483,8 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
LWLockRelease(ShmemIndexLock);
ereport(ERROR,
(errcode(ERRCODE_OUT_OF_MEMORY),
- errmsg("could not create ShmemIndex entry for data structure \"%s\"",
- name)));
+ errmsg("could not create ShmemIndex entry for data structure \"%s\" in slot %d",
+ name, shmem_slot)));
}
if (*foundPtr)
@@ -459,7 +509,7 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
Size allocated_size;
/* It isn't in the table yet. allocate and initialize it */
- structPtr = ShmemAllocRaw(size, &allocated_size);
+ structPtr = ShmemAllocRawInSlot(size, &allocated_size, shmem_slot);
if (structPtr == NULL)
{
/* out of memory; remove the failed ShmemIndex entry */
@@ -478,14 +528,13 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
LWLockRelease(ShmemIndexLock);
- Assert(ShmemAddrIsValid(structPtr));
+ Assert(ShmemAddrIsValidInSlot(structPtr, shmem_slot));
Assert(structPtr == (void *) CACHELINEALIGN(structPtr));
return structPtr;
}
-
/*
* Add two Size values, checking for overflow
*/
@@ -545,7 +594,7 @@ pg_get_shmem_allocations(PG_FUNCTION_ARGS)
while ((ent = (ShmemIndexEnt *) hash_seq_search(&hstat)) != NULL)
{
values[0] = CStringGetTextDatum(ent->key);
- values[1] = Int64GetDatum((char *) ent->location - (char *) ShmemSegHdr);
+ values[1] = Int64GetDatum((char *) ent->location - (char *) Segments[MAIN_SHMEM_SLOT].ShmemSegHdr);
values[2] = Int64GetDatum(ent->size);
values[3] = Int64GetDatum(ent->allocated_size);
named_allocated += ent->allocated_size;
@@ -557,15 +606,15 @@ pg_get_shmem_allocations(PG_FUNCTION_ARGS)
/* output shared memory allocated but not counted via the shmem index */
values[0] = CStringGetTextDatum("<anonymous>");
nulls[1] = true;
- values[2] = Int64GetDatum(ShmemSegHdr->freeoffset - named_allocated);
+ values[2] = Int64GetDatum(Segments[MAIN_SHMEM_SLOT].ShmemSegHdr->freeoffset - named_allocated);
values[3] = values[2];
tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls);
/* output as-of-yet unused shared memory */
nulls[0] = true;
- values[1] = Int64GetDatum(ShmemSegHdr->freeoffset);
+ values[1] = Int64GetDatum(Segments[MAIN_SHMEM_SLOT].ShmemSegHdr->freeoffset);
nulls[1] = false;
- values[2] = Int64GetDatum(ShmemSegHdr->totalsize - ShmemSegHdr->freeoffset);
+ values[2] = Int64GetDatum(Segments[MAIN_SHMEM_SLOT].ShmemSegHdr->totalsize - Segments[MAIN_SHMEM_SLOT].ShmemSegHdr->freeoffset);
values[3] = values[2];
tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls);
diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c
index e765754d80..fb0c33bf17 100644
--- a/src/backend/storage/lmgr/lwlock.c
+++ b/src/backend/storage/lmgr/lwlock.c
@@ -81,6 +81,7 @@
#include "pgstat.h"
#include "port/pg_bitutils.h"
#include "postmaster/postmaster.h"
+#include "storage/pg_shmem.h"
#include "storage/proc.h"
#include "storage/proclist.h"
#include "storage/spin.h"
@@ -607,9 +608,9 @@ LWLockNewTrancheId(void)
LWLockCounter = (int *) ((char *) MainLWLockArray - sizeof(int));
/* We use the ShmemLock spinlock to protect LWLockCounter */
- SpinLockAcquire(ShmemLock);
+ SpinLockAcquire(Segments[MAIN_SHMEM_SLOT].ShmemLock);
result = (*LWLockCounter)++;
- SpinLockRelease(ShmemLock);
+ SpinLockRelease(Segments[MAIN_SHMEM_SLOT].ShmemLock);
return result;
}
diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h
index f190e6e5e4..aef80e049b 100644
--- a/src/include/storage/buf_internals.h
+++ b/src/include/storage/buf_internals.h
@@ -23,6 +23,7 @@
#include "storage/latch.h"
#include "storage/lwlock.h"
#include "storage/shmem.h"
+#include "storage/pg_shmem.h"
#include "storage/smgr.h"
#include "storage/spin.h"
#include "utils/relcache.h"
diff --git a/src/include/storage/ipc.h b/src/include/storage/ipc.h
index b2d062781e..be4b131288 100644
--- a/src/include/storage/ipc.h
+++ b/src/include/storage/ipc.h
@@ -77,7 +77,7 @@ extern void check_on_shmem_exit_lists_are_empty(void);
/* ipci.c */
extern PGDLLIMPORT shmem_startup_hook_type shmem_startup_hook;
-extern Size CalculateShmemSize(int *num_semaphores);
+extern Size CalculateShmemSize(int *num_semaphores, int shmem_slot);
extern void CreateSharedMemoryAndSemaphores(void);
#ifdef EXEC_BACKEND
extern void AttachSharedMemoryStructs(void);
diff --git a/src/include/storage/pg_sema.h b/src/include/storage/pg_sema.h
index dfef79ac96..081fffaf16 100644
--- a/src/include/storage/pg_sema.h
+++ b/src/include/storage/pg_sema.h
@@ -41,7 +41,7 @@ typedef HANDLE PGSemaphore;
extern Size PGSemaphoreShmemSize(int maxSemas);
/* Module initialization (called during postmaster start or shmem reinit) */
-extern void PGReserveSemaphores(int maxSemas);
+extern void PGReserveSemaphores(int maxSemas, int shmem_slot);
/* Allocate a PGSemaphore structure with initial count 1 */
extern PGSemaphore PGSemaphoreCreate(void);
diff --git a/src/include/storage/pg_shmem.h b/src/include/storage/pg_shmem.h
index 3065ff5be7..e968deeef7 100644
--- a/src/include/storage/pg_shmem.h
+++ b/src/include/storage/pg_shmem.h
@@ -25,6 +25,7 @@
#define PG_SHMEM_H
#include "storage/dsm_impl.h"
+#include "storage/spin.h"
typedef struct PGShmemHeader /* standard header for all Postgres shmem */
{
@@ -41,6 +42,20 @@ typedef struct PGShmemHeader /* standard header for all Postgres shmem */
#endif
} PGShmemHeader;
+typedef struct ShmemSegment
+{
+ PGShmemHeader *ShmemSegHdr; /* shared mem segment header */
+ void *ShmemBase; /* start address of shared memory */
+ void *ShmemEnd; /* end+1 address of shared memory */
+ slock_t *ShmemLock; /* spinlock for shared memory and LWLock
+ * allocation */
+} ShmemSegment;
+
+// Number of available slots for anonymous memory mappings
+#define ANON_MAPPINGS 1
+
+extern PGDLLIMPORT ShmemSegment Segments[ANON_MAPPINGS];
+
/* GUC variables */
extern PGDLLIMPORT int shared_memory_type;
extern PGDLLIMPORT int huge_pages;
@@ -90,4 +105,7 @@ extern bool PGSharedMemoryIsInUse(unsigned long id1, unsigned long id2);
extern void PGSharedMemoryDetach(void);
extern void GetHugePageSize(Size *hugepagesize, int *mmap_flags);
+/* The main slot, contains everything except buffer blocks and related data. */
+#define MAIN_SHMEM_SLOT 0
+
#endif /* PG_SHMEM_H */
diff --git a/src/include/storage/shmem.h b/src/include/storage/shmem.h
index 842989111c..d3e9cc721d 100644
--- a/src/include/storage/shmem.h
+++ b/src/include/storage/shmem.h
@@ -28,15 +28,25 @@
/* shmem.c */
extern PGDLLIMPORT slock_t *ShmemLock;
extern void InitShmemAccess(void *seghdr);
+extern void InitShmemAccessInSlot(void *seghdr, int shmem_slot);
extern void InitShmemAllocation(void);
+extern void InitShmemAllocationInSlot(int shmem_slot);
extern void *ShmemAlloc(Size size);
+extern void *ShmemAllocInSlot(Size size, int shmem_slot);
extern void *ShmemAllocNoError(Size size);
extern void *ShmemAllocUnlocked(Size size);
+extern void *ShmemAllocUnlockedInSlot(Size size, int shmem_slot);
extern bool ShmemAddrIsValid(const void *addr);
+extern bool ShmemAddrIsValidInSlot(const void *addr, int shmem_slot);
extern void InitShmemIndex(void);
+extern void InitVariableShmemIndex(void);
extern HTAB *ShmemInitHash(const char *name, long init_size, long max_size,
HASHCTL *infoP, int hash_flags);
+extern HTAB *ShmemInitHashInSlot(const char *name, long init_size, long max_size,
+ HASHCTL *infoP, int hash_flags, int shmem_slot);
extern void *ShmemInitStruct(const char *name, Size size, bool *foundPtr);
+extern void *ShmemInitStructInSlot(const char *name, Size size, bool *foundPtr,
+ int shmem_slot);
extern Size add_size(Size s1, Size s2);
extern Size mul_size(Size s1, Size s2);
base-commit: 2488058dc356a43455b21a099ea879fff9266634
--
2.45.1
--ers4zrhjqnhlajas
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename="v1-0002-Allow-placing-shared-memory-mapping-with-an-offse.patch"
^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v12 3/4] Remove uses of popcount builtins.
@ 2026-02-06 16:00 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 9+ messages in thread
From: Nathan Bossart @ 2026-02-06 16:00 UTC (permalink / raw)
This commit replaces the implementations of pg_popcount{32,64} with
branchless ones in plain C. While these new implementations do not
make use of more sophisticated population count instructions
available on some CPUs, testing indicates they perform well,
especially now that they are inlined. A follow-up commit will
replace various loops over these functions with calls to
pg_popcount(), leaving us little reason to worry about
micro-optimizing them further.
Since this commit removes the only uses of the popcount builtins,
we can also remove the corresponding configuration checks.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
configure | 38 ----------------------------
configure.ac | 1 -
meson.build | 1 -
src/include/pg_config.h.in | 3 ---
src/include/port/pg_bitutils.h | 46 +++++++++++-----------------------
src/port/pg_popcount_aarch64.c | 5 ----
6 files changed, 14 insertions(+), 80 deletions(-)
diff --git a/configure b/configure
index a10a2c85c6a..8fe368b7201 100755
--- a/configure
+++ b/configure
@@ -15878,44 +15878,6 @@ cat >>confdefs.h <<_ACEOF
#define HAVE__BUILTIN_CTZ 1
_ACEOF
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __builtin_popcount" >&5
-$as_echo_n "checking for __builtin_popcount... " >&6; }
-if ${pgac_cv__builtin_popcount+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
-int
-call__builtin_popcount(unsigned int x)
-{
- return __builtin_popcount(x);
-}
-int
-main ()
-{
-
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- pgac_cv__builtin_popcount=yes
-else
- pgac_cv__builtin_popcount=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv__builtin_popcount" >&5
-$as_echo "$pgac_cv__builtin_popcount" >&6; }
-if test x"${pgac_cv__builtin_popcount}" = xyes ; then
-
-cat >>confdefs.h <<_ACEOF
-#define HAVE__BUILTIN_POPCOUNT 1
-_ACEOF
-
fi
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
diff --git a/configure.ac b/configure.ac
index 814e64a967e..f569b1c3f35 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1852,7 +1852,6 @@ PGAC_CHECK_BUILTIN_FUNC([__builtin_bswap64], [long int x])
# We assume that we needn't test all widths of these explicitly:
PGAC_CHECK_BUILTIN_FUNC([__builtin_clz], [unsigned int x])
PGAC_CHECK_BUILTIN_FUNC([__builtin_ctz], [unsigned int x])
-PGAC_CHECK_BUILTIN_FUNC([__builtin_popcount], [unsigned int x])
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
PGAC_CHECK_BUILTIN_FUNC_PTR([__builtin_frame_address], [0])
diff --git a/meson.build b/meson.build
index 96b3869df86..c89293cd80f 100644
--- a/meson.build
+++ b/meson.build
@@ -2004,7 +2004,6 @@ builtins = [
'ctz',
'constant_p',
'frame_address',
- 'popcount',
'unreachable',
]
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 339268dc8ef..2651b56ae4d 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -526,9 +526,6 @@
/* Define to 1 if your compiler understands __builtin_$op_overflow. */
#undef HAVE__BUILTIN_OP_OVERFLOW
-/* Define to 1 if your compiler understands __builtin_popcount. */
-#undef HAVE__BUILTIN_POPCOUNT
-
/* Define to 1 if your compiler understands __builtin_types_compatible_p. */
#undef HAVE__BUILTIN_TYPES_COMPATIBLE_P
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 789663edd93..c9b1f5f17dc 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -297,51 +297,33 @@ extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mas
/*
* pg_popcount32
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
*/
static inline int
pg_popcount32(uint32 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & 0x55555555;
+ word = (word & 0x33333333) + ((word >> 2) & 0x33333333);
+ return (((word + (word >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
}
/*
* pg_popcount64
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
*/
static inline int
pg_popcount64(uint64 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & UINT64CONST(0x5555555555555555);
+ word = (word & UINT64CONST(0x3333333333333333)) +
+ ((word >> 2) & UINT64CONST(0x3333333333333333));
+ word = (word + (word >> 4)) & UINT64CONST(0xf0f0f0f0f0f0f0f);
+ return (word * UINT64CONST(0x101010101010101)) >> 56;
}
/*
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index f474ef45510..b0f10ae07a4 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -298,11 +298,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
static inline int
pg_popcount64_neon(uint64 word)
{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
}
--
2.50.1 (Apple Git-155)
--AFLuD1w+kMRlv6Zk
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v12-0004-Make-use-of-pg_popcount-in-more-places.patch
^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v11 4/4] Remove uses of popcount builtins.
@ 2026-02-06 16:00 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 9+ messages in thread
From: Nathan Bossart @ 2026-02-06 16:00 UTC (permalink / raw)
---
configure | 38 ----------------------------------
configure.ac | 1 -
meson.build | 1 -
src/include/pg_config.h.in | 3 ---
src/include/port/pg_bitutils.h | 17 +--------------
src/port/pg_popcount_aarch64.c | 5 -----
6 files changed, 1 insertion(+), 64 deletions(-)
diff --git a/configure b/configure
index ba293931878..623aa397fae 100755
--- a/configure
+++ b/configure
@@ -15920,44 +15920,6 @@ cat >>confdefs.h <<_ACEOF
#define HAVE__BUILTIN_CTZ 1
_ACEOF
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __builtin_popcount" >&5
-$as_echo_n "checking for __builtin_popcount... " >&6; }
-if ${pgac_cv__builtin_popcount+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
-int
-call__builtin_popcount(unsigned int x)
-{
- return __builtin_popcount(x);
-}
-int
-main ()
-{
-
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- pgac_cv__builtin_popcount=yes
-else
- pgac_cv__builtin_popcount=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv__builtin_popcount" >&5
-$as_echo "$pgac_cv__builtin_popcount" >&6; }
-if test x"${pgac_cv__builtin_popcount}" = xyes ; then
-
-cat >>confdefs.h <<_ACEOF
-#define HAVE__BUILTIN_POPCOUNT 1
-_ACEOF
-
fi
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
diff --git a/configure.ac b/configure.ac
index 412fe358a2f..04c6a75bff7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1853,7 +1853,6 @@ PGAC_CHECK_BUILTIN_FUNC([__builtin_bswap64], [long int x])
# We assume that we needn't test all widths of these explicitly:
PGAC_CHECK_BUILTIN_FUNC([__builtin_clz], [unsigned int x])
PGAC_CHECK_BUILTIN_FUNC([__builtin_ctz], [unsigned int x])
-PGAC_CHECK_BUILTIN_FUNC([__builtin_popcount], [unsigned int x])
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
PGAC_CHECK_BUILTIN_FUNC_PTR([__builtin_frame_address], [0])
diff --git a/meson.build b/meson.build
index 0722b16927e..c607d8ac69a 100644
--- a/meson.build
+++ b/meson.build
@@ -2004,7 +2004,6 @@ builtins = [
'ctz',
'constant_p',
'frame_address',
- 'popcount',
'unreachable',
]
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index c089f2252c3..301328b8cd3 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -530,9 +530,6 @@
/* Define to 1 if your compiler understands __builtin_$op_overflow. */
#undef HAVE__BUILTIN_OP_OVERFLOW
-/* Define to 1 if your compiler understands __builtin_popcount. */
-#undef HAVE__BUILTIN_POPCOUNT
-
/* Define to 1 if your compiler understands __builtin_types_compatible_p. */
#undef HAVE__BUILTIN_TYPES_COMPATIBLE_P
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 3c58f6c6864..c9b1f5f17dc 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -313,32 +313,17 @@ pg_popcount32(uint32 word)
* pg_popcount64
* Return the number of 1 bits set in word
*
- * Plain C version adapted from
+ * Adapted from
* https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
*/
static inline int
pg_popcount64(uint64 word)
{
- /*
- * On x86, gcc generates a function call for this built-in unless the
- * popcnt instruction is available, so we use the plain C version in that
- * case to ensure inlining.
- */
-#if defined(HAVE__BUILTIN_POPCOUNT) && (defined(__POPCNT__) || !defined(__x86_64__))
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else
word -= (word >> 1) & UINT64CONST(0x5555555555555555);
word = (word & UINT64CONST(0x3333333333333333)) +
((word >> 2) & UINT64CONST(0x3333333333333333));
word = (word + (word >> 4)) & UINT64CONST(0xf0f0f0f0f0f0f0f);
return (word * UINT64CONST(0x101010101010101)) >> 56;
-#endif
}
/*
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index f474ef45510..b0f10ae07a4 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -298,11 +298,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
static inline int
pg_popcount64_neon(uint64 word)
{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
}
--
2.50.1 (Apple Git-155)
--mih0IfQp6StcW7xc--
^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v13 3/5] Remove uses of popcount builtins.
@ 2026-02-06 16:00 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 9+ messages in thread
From: Nathan Bossart @ 2026-02-06 16:00 UTC (permalink / raw)
This commit replaces the implementations of pg_popcount{32,64} with
branchless ones in plain C. While these new implementations do not
make use of more sophisticated population count instructions
available on some CPUs, testing indicates they perform well,
especially now that they are inlined. A follow-up commit will
replace various loops over these functions with calls to
pg_popcount(), leaving us little reason to worry about
micro-optimizing them further.
Since this commit removes the only uses of the popcount builtins,
we can also remove the corresponding configuration checks.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
configure | 38 ----------------------------
configure.ac | 1 -
meson.build | 1 -
src/include/pg_config.h.in | 3 ---
src/include/port/pg_bitutils.h | 46 +++++++++++-----------------------
src/port/pg_popcount_aarch64.c | 5 ----
6 files changed, 14 insertions(+), 80 deletions(-)
diff --git a/configure b/configure
index a10a2c85c6a..8fe368b7201 100755
--- a/configure
+++ b/configure
@@ -15878,44 +15878,6 @@ cat >>confdefs.h <<_ACEOF
#define HAVE__BUILTIN_CTZ 1
_ACEOF
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __builtin_popcount" >&5
-$as_echo_n "checking for __builtin_popcount... " >&6; }
-if ${pgac_cv__builtin_popcount+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
-int
-call__builtin_popcount(unsigned int x)
-{
- return __builtin_popcount(x);
-}
-int
-main ()
-{
-
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- pgac_cv__builtin_popcount=yes
-else
- pgac_cv__builtin_popcount=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv__builtin_popcount" >&5
-$as_echo "$pgac_cv__builtin_popcount" >&6; }
-if test x"${pgac_cv__builtin_popcount}" = xyes ; then
-
-cat >>confdefs.h <<_ACEOF
-#define HAVE__BUILTIN_POPCOUNT 1
-_ACEOF
-
fi
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
diff --git a/configure.ac b/configure.ac
index 814e64a967e..f569b1c3f35 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1852,7 +1852,6 @@ PGAC_CHECK_BUILTIN_FUNC([__builtin_bswap64], [long int x])
# We assume that we needn't test all widths of these explicitly:
PGAC_CHECK_BUILTIN_FUNC([__builtin_clz], [unsigned int x])
PGAC_CHECK_BUILTIN_FUNC([__builtin_ctz], [unsigned int x])
-PGAC_CHECK_BUILTIN_FUNC([__builtin_popcount], [unsigned int x])
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
PGAC_CHECK_BUILTIN_FUNC_PTR([__builtin_frame_address], [0])
diff --git a/meson.build b/meson.build
index 96b3869df86..c89293cd80f 100644
--- a/meson.build
+++ b/meson.build
@@ -2004,7 +2004,6 @@ builtins = [
'ctz',
'constant_p',
'frame_address',
- 'popcount',
'unreachable',
]
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 339268dc8ef..2651b56ae4d 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -526,9 +526,6 @@
/* Define to 1 if your compiler understands __builtin_$op_overflow. */
#undef HAVE__BUILTIN_OP_OVERFLOW
-/* Define to 1 if your compiler understands __builtin_popcount. */
-#undef HAVE__BUILTIN_POPCOUNT
-
/* Define to 1 if your compiler understands __builtin_types_compatible_p. */
#undef HAVE__BUILTIN_TYPES_COMPATIBLE_P
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 789663edd93..c9b1f5f17dc 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -297,51 +297,33 @@ extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mas
/*
* pg_popcount32
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
*/
static inline int
pg_popcount32(uint32 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & 0x55555555;
+ word = (word & 0x33333333) + ((word >> 2) & 0x33333333);
+ return (((word + (word >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
}
/*
* pg_popcount64
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
*/
static inline int
pg_popcount64(uint64 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & UINT64CONST(0x5555555555555555);
+ word = (word & UINT64CONST(0x3333333333333333)) +
+ ((word >> 2) & UINT64CONST(0x3333333333333333));
+ word = (word + (word >> 4)) & UINT64CONST(0xf0f0f0f0f0f0f0f);
+ return (word * UINT64CONST(0x101010101010101)) >> 56;
}
/*
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index f474ef45510..b0f10ae07a4 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -298,11 +298,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
static inline int
pg_popcount64_neon(uint64 word)
{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
}
--
2.50.1 (Apple Git-155)
--5NNwvjHYUnRaoEDo
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v13-0004-Convert-some-popcount-functions-to-macros.patch
^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v14 1/2] Remove uses of popcount builtins.
@ 2026-02-06 16:00 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 9+ messages in thread
From: Nathan Bossart @ 2026-02-06 16:00 UTC (permalink / raw)
This commit replaces the implementations of pg_popcount{32,64} with
branchless ones in plain C. While these new implementations do not
make use of more sophisticated population count instructions
available on some CPUs, testing indicates they perform well,
especially now that they are inlined. A follow-up commit will
replace various loops over these functions with calls to
pg_popcount(), leaving us little reason to worry about
micro-optimizing them further.
Since this commit removes the only uses of the popcount builtins,
we can also remove the corresponding configuration checks.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
configure | 38 ----------------------------
configure.ac | 1 -
meson.build | 1 -
src/include/pg_config.h.in | 3 ---
src/include/port/pg_bitutils.h | 46 +++++++++++-----------------------
src/port/pg_popcount_aarch64.c | 5 ----
6 files changed, 14 insertions(+), 80 deletions(-)
diff --git a/configure b/configure
index a10a2c85c6a..8fe368b7201 100755
--- a/configure
+++ b/configure
@@ -15878,44 +15878,6 @@ cat >>confdefs.h <<_ACEOF
#define HAVE__BUILTIN_CTZ 1
_ACEOF
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __builtin_popcount" >&5
-$as_echo_n "checking for __builtin_popcount... " >&6; }
-if ${pgac_cv__builtin_popcount+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
-int
-call__builtin_popcount(unsigned int x)
-{
- return __builtin_popcount(x);
-}
-int
-main ()
-{
-
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- pgac_cv__builtin_popcount=yes
-else
- pgac_cv__builtin_popcount=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv__builtin_popcount" >&5
-$as_echo "$pgac_cv__builtin_popcount" >&6; }
-if test x"${pgac_cv__builtin_popcount}" = xyes ; then
-
-cat >>confdefs.h <<_ACEOF
-#define HAVE__BUILTIN_POPCOUNT 1
-_ACEOF
-
fi
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
diff --git a/configure.ac b/configure.ac
index 814e64a967e..f569b1c3f35 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1852,7 +1852,6 @@ PGAC_CHECK_BUILTIN_FUNC([__builtin_bswap64], [long int x])
# We assume that we needn't test all widths of these explicitly:
PGAC_CHECK_BUILTIN_FUNC([__builtin_clz], [unsigned int x])
PGAC_CHECK_BUILTIN_FUNC([__builtin_ctz], [unsigned int x])
-PGAC_CHECK_BUILTIN_FUNC([__builtin_popcount], [unsigned int x])
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
PGAC_CHECK_BUILTIN_FUNC_PTR([__builtin_frame_address], [0])
diff --git a/meson.build b/meson.build
index 96b3869df86..c89293cd80f 100644
--- a/meson.build
+++ b/meson.build
@@ -2004,7 +2004,6 @@ builtins = [
'ctz',
'constant_p',
'frame_address',
- 'popcount',
'unreachable',
]
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 339268dc8ef..2651b56ae4d 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -526,9 +526,6 @@
/* Define to 1 if your compiler understands __builtin_$op_overflow. */
#undef HAVE__BUILTIN_OP_OVERFLOW
-/* Define to 1 if your compiler understands __builtin_popcount. */
-#undef HAVE__BUILTIN_POPCOUNT
-
/* Define to 1 if your compiler understands __builtin_types_compatible_p. */
#undef HAVE__BUILTIN_TYPES_COMPATIBLE_P
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 789663edd93..c9b1f5f17dc 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -297,51 +297,33 @@ extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mas
/*
* pg_popcount32
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
*/
static inline int
pg_popcount32(uint32 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & 0x55555555;
+ word = (word & 0x33333333) + ((word >> 2) & 0x33333333);
+ return (((word + (word >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
}
/*
* pg_popcount64
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
*/
static inline int
pg_popcount64(uint64 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & UINT64CONST(0x5555555555555555);
+ word = (word & UINT64CONST(0x3333333333333333)) +
+ ((word >> 2) & UINT64CONST(0x3333333333333333));
+ word = (word + (word >> 4)) & UINT64CONST(0xf0f0f0f0f0f0f0f);
+ return (word * UINT64CONST(0x101010101010101)) >> 56;
}
/*
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index f474ef45510..b0f10ae07a4 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -298,11 +298,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
static inline int
pg_popcount64_neon(uint64 word)
{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
}
--
2.50.1 (Apple Git-155)
--rjsRYmGG7r/fFdxn
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v14-0002-Make-use-of-pg_popcount-in-more-places.patch
^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v14 1/2] Remove uses of popcount builtins.
@ 2026-02-06 16:00 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 9+ messages in thread
From: Nathan Bossart @ 2026-02-06 16:00 UTC (permalink / raw)
This commit replaces the implementations of pg_popcount{32,64} with
branchless ones in plain C. While these new implementations do not
make use of more sophisticated population count instructions
available on some CPUs, testing indicates they perform well,
especially now that they are inlined. A follow-up commit will
replace various loops over these functions with calls to
pg_popcount(), leaving us little reason to worry about
micro-optimizing them further.
Since this commit removes the only uses of the popcount builtins,
we can also remove the corresponding configuration checks.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
configure | 38 ----------------------------
configure.ac | 1 -
meson.build | 1 -
src/include/pg_config.h.in | 3 ---
src/include/port/pg_bitutils.h | 46 +++++++++++-----------------------
src/port/pg_popcount_aarch64.c | 5 ----
6 files changed, 14 insertions(+), 80 deletions(-)
diff --git a/configure b/configure
index a10a2c85c6a..8fe368b7201 100755
--- a/configure
+++ b/configure
@@ -15878,44 +15878,6 @@ cat >>confdefs.h <<_ACEOF
#define HAVE__BUILTIN_CTZ 1
_ACEOF
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __builtin_popcount" >&5
-$as_echo_n "checking for __builtin_popcount... " >&6; }
-if ${pgac_cv__builtin_popcount+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
-int
-call__builtin_popcount(unsigned int x)
-{
- return __builtin_popcount(x);
-}
-int
-main ()
-{
-
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- pgac_cv__builtin_popcount=yes
-else
- pgac_cv__builtin_popcount=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv__builtin_popcount" >&5
-$as_echo "$pgac_cv__builtin_popcount" >&6; }
-if test x"${pgac_cv__builtin_popcount}" = xyes ; then
-
-cat >>confdefs.h <<_ACEOF
-#define HAVE__BUILTIN_POPCOUNT 1
-_ACEOF
-
fi
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
diff --git a/configure.ac b/configure.ac
index 814e64a967e..f569b1c3f35 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1852,7 +1852,6 @@ PGAC_CHECK_BUILTIN_FUNC([__builtin_bswap64], [long int x])
# We assume that we needn't test all widths of these explicitly:
PGAC_CHECK_BUILTIN_FUNC([__builtin_clz], [unsigned int x])
PGAC_CHECK_BUILTIN_FUNC([__builtin_ctz], [unsigned int x])
-PGAC_CHECK_BUILTIN_FUNC([__builtin_popcount], [unsigned int x])
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
PGAC_CHECK_BUILTIN_FUNC_PTR([__builtin_frame_address], [0])
diff --git a/meson.build b/meson.build
index 96b3869df86..c89293cd80f 100644
--- a/meson.build
+++ b/meson.build
@@ -2004,7 +2004,6 @@ builtins = [
'ctz',
'constant_p',
'frame_address',
- 'popcount',
'unreachable',
]
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 339268dc8ef..2651b56ae4d 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -526,9 +526,6 @@
/* Define to 1 if your compiler understands __builtin_$op_overflow. */
#undef HAVE__BUILTIN_OP_OVERFLOW
-/* Define to 1 if your compiler understands __builtin_popcount. */
-#undef HAVE__BUILTIN_POPCOUNT
-
/* Define to 1 if your compiler understands __builtin_types_compatible_p. */
#undef HAVE__BUILTIN_TYPES_COMPATIBLE_P
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 789663edd93..c9b1f5f17dc 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -297,51 +297,33 @@ extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mas
/*
* pg_popcount32
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
*/
static inline int
pg_popcount32(uint32 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & 0x55555555;
+ word = (word & 0x33333333) + ((word >> 2) & 0x33333333);
+ return (((word + (word >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
}
/*
* pg_popcount64
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
*/
static inline int
pg_popcount64(uint64 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & UINT64CONST(0x5555555555555555);
+ word = (word & UINT64CONST(0x3333333333333333)) +
+ ((word >> 2) & UINT64CONST(0x3333333333333333));
+ word = (word + (word >> 4)) & UINT64CONST(0xf0f0f0f0f0f0f0f);
+ return (word * UINT64CONST(0x101010101010101)) >> 56;
}
/*
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index f474ef45510..b0f10ae07a4 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -298,11 +298,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
static inline int
pg_popcount64_neon(uint64 word)
{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
}
--
2.50.1 (Apple Git-155)
--rjsRYmGG7r/fFdxn
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v14-0002-Make-use-of-pg_popcount-in-more-places.patch
^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v15 1/2] Remove uses of popcount builtins.
@ 2026-02-20 20:33 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 9+ messages in thread
From: Nathan Bossart @ 2026-02-20 20:33 UTC (permalink / raw)
This commit replaces the implementations of pg_popcount{32,64} with
branchless ones in plain C. Newer versions of popular compilers
will automatically replace these with more sophisticated population
count instructions if possible, leaving us little reason to
continue using the builtins. This also allows us to remove the
remaining architecture-specific implementations of pg_popcount64(),
which were only used by the corresponding architecture-specific
implementations of pg_popcount() and pg_popcount_masked(). Since
this commit removes the only uses of the popcount builtins, we can
remove the corresponding configuration checks, too.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
config/c-compiler.m4 | 26 +++++++
configure | 119 +++++++++++++--------------------
configure.ac | 23 +++----
meson.build | 43 +++++++-----
src/include/pg_config.h.in | 7 +-
src/include/port/pg_bitutils.h | 56 +++++++---------
src/port/pg_bitutils.c | 4 +-
src/port/pg_popcount_aarch64.c | 19 +-----
src/port/pg_popcount_x86.c | 27 ++------
9 files changed, 143 insertions(+), 181 deletions(-)
diff --git a/config/c-compiler.m4 b/config/c-compiler.m4
index 1509dbfa2ab..93fd6b3b617 100644
--- a/config/c-compiler.m4
+++ b/config/c-compiler.m4
@@ -743,6 +743,32 @@ fi
undefine([Ac_cachevar])dnl
])# PGAC_XSAVE_INTRINSICS
+# PGAC_X86_POPCNT_INTRINSICS
+# ----------------------
+# Check if the compiler supports the x86 POPCNT instructions, using the
+# _popcnt64 intrinsic function.
+#
+# If the instrinsic is supported, sets pgac_x86_popcnt_intrinsics
+AC_DEFUN([PGAC_X86_POPCNT_INTRINSICS],
+[define([Ac_cachevar], [AS_TR_SH([pgac_cv_x86_popcnt_intrinsics])])dnl
+AC_CACHE_CHECK([for _popcnt64], [Ac_cachevar],
+[AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <immintrin.h>
+ #if defined(__has_attribute) && __has_attribute (target)
+ __attribute__((target("popcnt")))
+ #endif
+ static int x86_popcnt_test(void)
+ {
+ return _popcnt64(0);
+ }],
+ [return x86_popcnt_test();])],
+ [Ac_cachevar=yes],
+ [Ac_cachevar=no])])
+if test x"$Ac_cachevar" = x"yes"; then
+ pgac_x86_popcnt_intrinsics=yes
+fi
+undefine([Ac_cachevar])dnl
+])# PGAC_X86_POPCNT_INTRINSICS
+
# PGAC_AVX512_POPCNT_INTRINSICS
# -----------------------------
# Check if the compiler supports the AVX-512 popcount instructions using the
diff --git a/configure b/configure
index e1a08129974..a37b8d9676c 100755
--- a/configure
+++ b/configure
@@ -15256,40 +15256,6 @@ fi
case $host_cpu in
- x86_64)
- # On x86_64, check if we can compile a popcntq instruction
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether assembler supports x86_64 popcntq" >&5
-$as_echo_n "checking whether assembler supports x86_64 popcntq... " >&6; }
-if ${pgac_cv_have_x86_64_popcntq+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
-int
-main ()
-{
-long long x = 1; long long r;
- __asm__ __volatile__ (" popcntq %1,%0\n" : "=q"(r) : "rm"(x));
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
- pgac_cv_have_x86_64_popcntq=yes
-else
- pgac_cv_have_x86_64_popcntq=no
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_have_x86_64_popcntq" >&5
-$as_echo "$pgac_cv_have_x86_64_popcntq" >&6; }
- if test x"$pgac_cv_have_x86_64_popcntq" = xyes ; then
-
-$as_echo "#define HAVE_X86_64_POPCNTQ 1" >>confdefs.h
-
- fi
- ;;
ppc*|powerpc*)
# On PPC, check if compiler accepts "i"(x) when __builtin_constant_p(x).
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether __builtin_constant_p(x) implies \"i\"(x) acceptance" >&5
@@ -15836,44 +15802,6 @@ cat >>confdefs.h <<_ACEOF
#define HAVE__BUILTIN_CTZ 1
_ACEOF
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __builtin_popcount" >&5
-$as_echo_n "checking for __builtin_popcount... " >&6; }
-if ${pgac_cv__builtin_popcount+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
-int
-call__builtin_popcount(unsigned int x)
-{
- return __builtin_popcount(x);
-}
-int
-main ()
-{
-
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- pgac_cv__builtin_popcount=yes
-else
- pgac_cv__builtin_popcount=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv__builtin_popcount" >&5
-$as_echo "$pgac_cv__builtin_popcount" >&6; }
-if test x"${pgac_cv__builtin_popcount}" = xyes ; then
-
-cat >>confdefs.h <<_ACEOF
-#define HAVE__BUILTIN_POPCOUNT 1
-_ACEOF
-
fi
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
@@ -17721,6 +17649,53 @@ $as_echo "#define HAVE_XSAVE_INTRINSICS 1" >>confdefs.h
fi
+# Check for x86 POPCNT intrinsics
+#
+if test x"$host_cpu" = x"x86_64"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for _popcnt64" >&5
+$as_echo_n "checking for _popcnt64... " >&6; }
+if ${pgac_cv_x86_popcnt_intrinsics+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <immintrin.h>
+ #if defined(__has_attribute) && __has_attribute (target)
+ __attribute__((target("popcnt")))
+ #endif
+ static int x86_popcnt_test(void)
+ {
+ return _popcnt64(0);
+ }
+int
+main ()
+{
+return x86_popcnt_test();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ pgac_cv_x86_popcnt_intrinsics=yes
+else
+ pgac_cv_x86_popcnt_intrinsics=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_x86_popcnt_intrinsics" >&5
+$as_echo "$pgac_cv_x86_popcnt_intrinsics" >&6; }
+if test x"$pgac_cv_x86_popcnt_intrinsics" = x"yes"; then
+ pgac_x86_popcnt_intrinsics=yes
+fi
+
+ if test x"$pgac_x86_popcnt_intrinsics" = x"yes"; then
+
+$as_echo "#define HAVE_X86_POPCNT_INTRINSICS 1" >>confdefs.h
+
+ fi
+fi
+
# Check for AVX-512 popcount intrinsics
#
if test x"$host_cpu" = x"x86_64"; then
diff --git a/configure.ac b/configure.ac
index cc85c233c03..1594ee802b8 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1746,19 +1746,6 @@ AC_CHECK_TYPES([struct option], [], [],
#endif])
case $host_cpu in
- x86_64)
- # On x86_64, check if we can compile a popcntq instruction
- AC_CACHE_CHECK([whether assembler supports x86_64 popcntq],
- [pgac_cv_have_x86_64_popcntq],
- [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],
- [long long x = 1; long long r;
- __asm__ __volatile__ (" popcntq %1,%0\n" : "=q"(r) : "rm"(x));])],
- [pgac_cv_have_x86_64_popcntq=yes],
- [pgac_cv_have_x86_64_popcntq=no])])
- if test x"$pgac_cv_have_x86_64_popcntq" = xyes ; then
- AC_DEFINE(HAVE_X86_64_POPCNTQ, 1, [Define to 1 if the assembler supports X86_64's POPCNTQ instruction.])
- fi
- ;;
ppc*|powerpc*)
# On PPC, check if compiler accepts "i"(x) when __builtin_constant_p(x).
AC_CACHE_CHECK([whether __builtin_constant_p(x) implies "i"(x) acceptance],
@@ -1851,7 +1838,6 @@ PGAC_CHECK_BUILTIN_FUNC([__builtin_bswap64], [long int x])
# We assume that we needn't test all widths of these explicitly:
PGAC_CHECK_BUILTIN_FUNC([__builtin_clz], [unsigned int x])
PGAC_CHECK_BUILTIN_FUNC([__builtin_ctz], [unsigned int x])
-PGAC_CHECK_BUILTIN_FUNC([__builtin_popcount], [unsigned int x])
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
PGAC_CHECK_BUILTIN_FUNC_PTR([__builtin_frame_address], [0])
@@ -2128,6 +2114,15 @@ if test x"$pgac_xsave_intrinsics" = x"yes"; then
AC_DEFINE(HAVE_XSAVE_INTRINSICS, 1, [Define to 1 if you have XSAVE intrinsics.])
fi
+# Check for x86 POPCNT intrinsics
+#
+if test x"$host_cpu" = x"x86_64"; then
+ PGAC_X86_POPCNT_INTRINSICS()
+ if test x"$pgac_x86_popcnt_intrinsics" = x"yes"; then
+ AC_DEFINE(HAVE_X86_POPCNT_INTRINSICS, 1, [Define to 1 if you have x86 POPCNT intrinsics.])
+ fi
+fi
+
# Check for AVX-512 popcount intrinsics
#
if test x"$host_cpu" = x"x86_64"; then
diff --git a/meson.build b/meson.build
index 055e96315d0..b44cfd3993a 100644
--- a/meson.build
+++ b/meson.build
@@ -2006,7 +2006,6 @@ builtins = [
'ctz',
'constant_p',
'frame_address',
- 'popcount',
'unreachable',
]
@@ -2377,6 +2376,31 @@ int main(void)
endif
+###############################################################
+# Check for the availability of x86 POPCNT intrinsics.
+###############################################################
+
+if host_cpu == 'x86_64'
+
+ prog = '''
+#include <immintrin.h>
+
+#if defined(__has_attribute) && __has_attribute (target)
+__attribute__((target("popcnt")))
+#endif
+int main(void)
+{
+ return _popcnt64(0);
+}
+'''
+
+ if cc.links(prog, name: 'x86 POPCNT intrinsics', args: test_c_args)
+ cdata.set('HAVE_X86_POPCNT_INTRINSICS', 1)
+ endif
+
+endif
+
+
###############################################################
# Check for the availability of AVX-512 popcount intrinsics.
###############################################################
@@ -2641,22 +2665,7 @@ endif
# Other CPU specific stuff
###############################################################
-if host_cpu == 'x86_64'
-
- if cc.get_id() == 'msvc'
- cdata.set('HAVE_X86_64_POPCNTQ', 1)
- elif cc.compiles('''
- void main(void)
- {
- long long x = 1; long long r;
- __asm__ __volatile__ (" popcntq %1,%0\n" : "=q"(r) : "rm"(x));
- }''',
- name: '@0@: popcntq instruction'.format(host_cpu),
- args: test_c_args)
- cdata.set('HAVE_X86_64_POPCNTQ', 1)
- endif
-
-elif host_cpu == 'ppc' or host_cpu == 'ppc64'
+if host_cpu == 'ppc' or host_cpu == 'ppc64'
# Check if compiler accepts "i"(x) when __builtin_constant_p(x).
if cdata.has('HAVE__BUILTIN_CONSTANT_P')
if cc.compiles('''
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 3824a5571bb..2f6d291d9b2 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -493,8 +493,8 @@
/* Define to 1 if you have the `X509_get_signature_info' function. */
#undef HAVE_X509_GET_SIGNATURE_INFO
-/* Define to 1 if the assembler supports X86_64's POPCNTQ instruction. */
-#undef HAVE_X86_64_POPCNTQ
+/* Define to 1 if you have x86 POPCNT intrinsics. */
+#undef HAVE_X86_POPCNT_INTRINSICS
/* Define to 1 if you have the <xlocale.h> header file. */
#undef HAVE_XLOCALE_H
@@ -526,9 +526,6 @@
/* Define to 1 if your compiler understands __builtin_$op_overflow. */
#undef HAVE__BUILTIN_OP_OVERFLOW
-/* Define to 1 if your compiler understands __builtin_popcount. */
-#undef HAVE__BUILTIN_POPCOUNT
-
/* Define to 1 if your compiler understands __builtin_types_compatible_p. */
#undef HAVE__BUILTIN_TYPES_COMPATIBLE_P
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 789663edd93..53df0594823 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -279,7 +279,7 @@ pg_ceil_log2_64(uint64 num)
extern uint64 pg_popcount_portable(const char *buf, int bytes);
extern uint64 pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask);
-#if defined(HAVE_X86_64_POPCNTQ) || defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
+#if defined(HAVE_X86_POPCNT_INTRINSICS) || defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
/*
* Attempt to use specialized CPU instructions, but perform a runtime check
* first.
@@ -297,51 +297,41 @@ extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mas
/*
* pg_popcount32
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ *
+ * Note that newer versions of popular compilers will automatically replace
+ * this with a special popcount instruction if possible, so there isn't much
+ * reason to use builtin functions or intrinsics.
*/
static inline int
pg_popcount32(uint32 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & 0x55555555;
+ word = (word & 0x33333333) + ((word >> 2) & 0x33333333);
+ return (((word + (word >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
}
/*
* pg_popcount64
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ *
+ * Note that newer versions of popular compilers will automatically replace
+ * this with a special popcount instruction if possible, so there isn't much
+ * reason to use builtin functions or intrinsics.
*/
static inline int
pg_popcount64(uint64 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & UINT64CONST(0x5555555555555555);
+ word = (word & UINT64CONST(0x3333333333333333)) +
+ ((word >> 2) & UINT64CONST(0x3333333333333333));
+ word = (word + (word >> 4)) & UINT64CONST(0xf0f0f0f0f0f0f0f);
+ return (word * UINT64CONST(0x101010101010101)) >> 56;
}
/*
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c
index 49b130f1306..71327810119 100644
--- a/src/port/pg_bitutils.c
+++ b/src/port/pg_bitutils.c
@@ -162,7 +162,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
return popcnt;
}
-#if !defined(HAVE_X86_64_POPCNTQ) && !defined(USE_NEON)
+#if !defined(HAVE_X86_POPCNT_INTRINSICS) && !defined(USE_NEON)
/*
* When special CPU instructions are not available, there's no point in using
@@ -191,4 +191,4 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
return pg_popcount_masked_portable(buf, bytes, mask);
}
-#endif /* ! HAVE_X86_64_POPCNTQ && ! USE_NEON */
+#endif /* ! HAVE_X86_POPCNT_INTRINSICS && ! USE_NEON */
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index f474ef45510..74f71593721 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -291,21 +291,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
#endif /* ! USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount64_neon
- * Return number of 1 bits in word
- */
-static inline int
-pg_popcount64_neon(uint64 word)
-{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
- return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
-}
-
/*
* pg_popcount_neon
* Returns number of 1 bits in buf
@@ -373,7 +358,7 @@ pg_popcount_neon(const char *buf, int bytes)
*/
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
{
- popcnt += pg_popcount64_neon(*((const uint64 *) buf));
+ popcnt += pg_popcount64(*((const uint64 *) buf));
buf += sizeof(uint64);
}
@@ -455,7 +440,7 @@ pg_popcount_masked_neon(const char *buf, int bytes, bits8 mask)
*/
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
{
- popcnt += pg_popcount64_neon(*((const uint64 *) buf) & mask64);
+ popcnt += pg_popcount64(*((const uint64 *) buf) & mask64);
buf += sizeof(uint64);
}
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.c
index 6bce089432f..45ade1ee37b 100644
--- a/src/port/pg_popcount_x86.c
+++ b/src/port/pg_popcount_x86.c
@@ -12,7 +12,7 @@
*/
#include "c.h"
-#ifdef HAVE_X86_64_POPCNTQ
+#ifdef HAVE_X86_POPCNT_INTRINSICS
#if defined(HAVE__GET_CPUID) || defined(HAVE__GET_CPUID_COUNT)
#include <cpuid.h>
@@ -314,28 +314,12 @@ pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount64_sse42
- * Return the number of 1 bits set in word
- */
-static inline int
-pg_popcount64_sse42(uint64 word)
-{
-#ifdef _MSC_VER
- return __popcnt64(word);
-#else
- uint64 res;
-
-__asm__ __volatile__(" popcntq %1,%0\n":"=q"(res):"rm"(word):"cc");
- return (int) res;
-#endif
-}
-
/*
* pg_popcount_sse42
* Returns the number of 1-bits in buf
*/
pg_attribute_no_sanitize_alignment()
+pg_attribute_target("popcnt")
static uint64
pg_popcount_sse42(const char *buf, int bytes)
{
@@ -344,7 +328,7 @@ pg_popcount_sse42(const char *buf, int bytes)
while (bytes >= 8)
{
- popcnt += pg_popcount64_sse42(*words++);
+ popcnt += pg_popcount64(*words++);
bytes -= 8;
}
@@ -362,6 +346,7 @@ pg_popcount_sse42(const char *buf, int bytes)
* Returns the number of 1-bits in buf after applying the mask to each byte
*/
pg_attribute_no_sanitize_alignment()
+pg_attribute_target("popcnt")
static uint64
pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask)
{
@@ -371,7 +356,7 @@ pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask)
while (bytes >= 8)
{
- popcnt += pg_popcount64_sse42(*words++ & maskv);
+ popcnt += pg_popcount64(*words++ & maskv);
bytes -= 8;
}
@@ -384,4 +369,4 @@ pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask)
return popcnt;
}
-#endif /* HAVE_X86_64_POPCNTQ */
+#endif /* HAVE_X86_POPCNT_INTRINSICS */
--
2.50.1 (Apple Git-155)
--PPBl4GgF05zqfOFU
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v15-0002-Make-use-of-pg_popcount-in-more-places.patch
^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v16 1/2] Remove uses of popcount builtins.
@ 2026-02-21 21:12 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 9+ messages in thread
From: Nathan Bossart @ 2026-02-21 21:12 UTC (permalink / raw)
This commit replaces the implementations of pg_popcount{32,64} with
branchless ones in plain C. While these new implementations do not
make use of more sophisticated population count instructions
available on some CPUs, testing indicates they perform well,
especially now that they are inlined. Newer versions of popular
compilers will automatically replace these with special
instructions if possible, anyway. A follow-up commit will replace
various loops over these functions with calls to pg_popcount(),
leaving us little reason to worry about micro-optimizing them
further.
Since this commit removes the only uses of the popcount builtins,
we can also remove the corresponding configuration checks.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
configure | 38 ------------------------
configure.ac | 1 -
meson.build | 1 -
src/include/pg_config.h.in | 3 --
src/include/port/pg_bitutils.h | 54 ++++++++++++++--------------------
src/port/pg_popcount_aarch64.c | 5 ----
6 files changed, 22 insertions(+), 80 deletions(-)
diff --git a/configure b/configure
index e1a08129974..cb143a48141 100755
--- a/configure
+++ b/configure
@@ -15836,44 +15836,6 @@ cat >>confdefs.h <<_ACEOF
#define HAVE__BUILTIN_CTZ 1
_ACEOF
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __builtin_popcount" >&5
-$as_echo_n "checking for __builtin_popcount... " >&6; }
-if ${pgac_cv__builtin_popcount+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
-int
-call__builtin_popcount(unsigned int x)
-{
- return __builtin_popcount(x);
-}
-int
-main ()
-{
-
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- pgac_cv__builtin_popcount=yes
-else
- pgac_cv__builtin_popcount=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv__builtin_popcount" >&5
-$as_echo "$pgac_cv__builtin_popcount" >&6; }
-if test x"${pgac_cv__builtin_popcount}" = xyes ; then
-
-cat >>confdefs.h <<_ACEOF
-#define HAVE__BUILTIN_POPCOUNT 1
-_ACEOF
-
fi
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
diff --git a/configure.ac b/configure.ac
index cc85c233c03..3951787313a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1851,7 +1851,6 @@ PGAC_CHECK_BUILTIN_FUNC([__builtin_bswap64], [long int x])
# We assume that we needn't test all widths of these explicitly:
PGAC_CHECK_BUILTIN_FUNC([__builtin_clz], [unsigned int x])
PGAC_CHECK_BUILTIN_FUNC([__builtin_ctz], [unsigned int x])
-PGAC_CHECK_BUILTIN_FUNC([__builtin_popcount], [unsigned int x])
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
PGAC_CHECK_BUILTIN_FUNC_PTR([__builtin_frame_address], [0])
diff --git a/meson.build b/meson.build
index 055e96315d0..e0972f3a3d9 100644
--- a/meson.build
+++ b/meson.build
@@ -2006,7 +2006,6 @@ builtins = [
'ctz',
'constant_p',
'frame_address',
- 'popcount',
'unreachable',
]
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 3824a5571bb..af08c5a7eb8 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -526,9 +526,6 @@
/* Define to 1 if your compiler understands __builtin_$op_overflow. */
#undef HAVE__BUILTIN_OP_OVERFLOW
-/* Define to 1 if your compiler understands __builtin_popcount. */
-#undef HAVE__BUILTIN_POPCOUNT
-
/* Define to 1 if your compiler understands __builtin_types_compatible_p. */
#undef HAVE__BUILTIN_TYPES_COMPATIBLE_P
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 789663edd93..0bca559caaa 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -297,51 +297,41 @@ extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mas
/*
* pg_popcount32
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ *
+ * Note that newer versions of popular compilers will automatically replace
+ * this with a special popcount instruction if possible, so we don't bother
+ * using builtin functions or intrinsics.
*/
static inline int
pg_popcount32(uint32 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & 0x55555555;
+ word = (word & 0x33333333) + ((word >> 2) & 0x33333333);
+ return (((word + (word >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
}
/*
* pg_popcount64
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ *
+ * Note that newer versions of popular compilers will automatically replace
+ * this with a special popcount instruction if possible, so we don't bother
+ * using builtin functions or intrinsics.
*/
static inline int
pg_popcount64(uint64 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & UINT64CONST(0x5555555555555555);
+ word = (word & UINT64CONST(0x3333333333333333)) +
+ ((word >> 2) & UINT64CONST(0x3333333333333333));
+ word = (word + (word >> 4)) & UINT64CONST(0xf0f0f0f0f0f0f0f);
+ return (word * UINT64CONST(0x101010101010101)) >> 56;
}
/*
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index f474ef45510..b0f10ae07a4 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -298,11 +298,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
static inline int
pg_popcount64_neon(uint64 word)
{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
}
--
2.50.1 (Apple Git-155)
--/1paUjurV8mGDyTY
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v16-0002-Make-use-of-pg_popcount-in-more-places.patch
^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v16 1/2] Remove uses of popcount builtins.
@ 2026-02-21 21:12 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 9+ messages in thread
From: Nathan Bossart @ 2026-02-21 21:12 UTC (permalink / raw)
This commit replaces the implementations of pg_popcount{32,64} with
branchless ones in plain C. While these new implementations do not
make use of more sophisticated population count instructions
available on some CPUs, testing indicates they perform well,
especially now that they are inlined. Newer versions of popular
compilers will automatically replace these with special
instructions if possible, anyway. A follow-up commit will replace
various loops over these functions with calls to pg_popcount(),
leaving us little reason to worry about micro-optimizing them
further.
Since this commit removes the only uses of the popcount builtins,
we can also remove the corresponding configuration checks.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
configure | 38 ------------------------
configure.ac | 1 -
meson.build | 1 -
src/include/pg_config.h.in | 3 --
src/include/port/pg_bitutils.h | 54 ++++++++++++++--------------------
src/port/pg_popcount_aarch64.c | 5 ----
6 files changed, 22 insertions(+), 80 deletions(-)
diff --git a/configure b/configure
index e1a08129974..cb143a48141 100755
--- a/configure
+++ b/configure
@@ -15836,44 +15836,6 @@ cat >>confdefs.h <<_ACEOF
#define HAVE__BUILTIN_CTZ 1
_ACEOF
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __builtin_popcount" >&5
-$as_echo_n "checking for __builtin_popcount... " >&6; }
-if ${pgac_cv__builtin_popcount+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
-int
-call__builtin_popcount(unsigned int x)
-{
- return __builtin_popcount(x);
-}
-int
-main ()
-{
-
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- pgac_cv__builtin_popcount=yes
-else
- pgac_cv__builtin_popcount=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv__builtin_popcount" >&5
-$as_echo "$pgac_cv__builtin_popcount" >&6; }
-if test x"${pgac_cv__builtin_popcount}" = xyes ; then
-
-cat >>confdefs.h <<_ACEOF
-#define HAVE__BUILTIN_POPCOUNT 1
-_ACEOF
-
fi
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
diff --git a/configure.ac b/configure.ac
index cc85c233c03..3951787313a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1851,7 +1851,6 @@ PGAC_CHECK_BUILTIN_FUNC([__builtin_bswap64], [long int x])
# We assume that we needn't test all widths of these explicitly:
PGAC_CHECK_BUILTIN_FUNC([__builtin_clz], [unsigned int x])
PGAC_CHECK_BUILTIN_FUNC([__builtin_ctz], [unsigned int x])
-PGAC_CHECK_BUILTIN_FUNC([__builtin_popcount], [unsigned int x])
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
PGAC_CHECK_BUILTIN_FUNC_PTR([__builtin_frame_address], [0])
diff --git a/meson.build b/meson.build
index 055e96315d0..e0972f3a3d9 100644
--- a/meson.build
+++ b/meson.build
@@ -2006,7 +2006,6 @@ builtins = [
'ctz',
'constant_p',
'frame_address',
- 'popcount',
'unreachable',
]
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 3824a5571bb..af08c5a7eb8 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -526,9 +526,6 @@
/* Define to 1 if your compiler understands __builtin_$op_overflow. */
#undef HAVE__BUILTIN_OP_OVERFLOW
-/* Define to 1 if your compiler understands __builtin_popcount. */
-#undef HAVE__BUILTIN_POPCOUNT
-
/* Define to 1 if your compiler understands __builtin_types_compatible_p. */
#undef HAVE__BUILTIN_TYPES_COMPATIBLE_P
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 789663edd93..0bca559caaa 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -297,51 +297,41 @@ extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mas
/*
* pg_popcount32
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ *
+ * Note that newer versions of popular compilers will automatically replace
+ * this with a special popcount instruction if possible, so we don't bother
+ * using builtin functions or intrinsics.
*/
static inline int
pg_popcount32(uint32 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & 0x55555555;
+ word = (word & 0x33333333) + ((word >> 2) & 0x33333333);
+ return (((word + (word >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
}
/*
* pg_popcount64
* Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ *
+ * Note that newer versions of popular compilers will automatically replace
+ * this with a special popcount instruction if possible, so we don't bother
+ * using builtin functions or intrinsics.
*/
static inline int
pg_popcount64(uint64 word)
{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
+ word -= (word >> 1) & UINT64CONST(0x5555555555555555);
+ word = (word & UINT64CONST(0x3333333333333333)) +
+ ((word >> 2) & UINT64CONST(0x3333333333333333));
+ word = (word + (word >> 4)) & UINT64CONST(0xf0f0f0f0f0f0f0f);
+ return (word * UINT64CONST(0x101010101010101)) >> 56;
}
/*
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index f474ef45510..b0f10ae07a4 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -298,11 +298,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
static inline int
pg_popcount64_neon(uint64 word)
{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
}
--
2.50.1 (Apple Git-155)
--/1paUjurV8mGDyTY
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v16-0002-Make-use-of-pg_popcount-in-more-places.patch
^ permalink raw reply [nested|flat] 9+ messages in thread
end of thread, other threads:[~2026-02-21 21:12 UTC | newest]
Thread overview: 9+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2024-10-09 13:41 [PATCH v1 1/5] Allow to use multiple shared memory mappings Dmitrii Dolgov <9erthalion6@gmail.com>
2026-02-06 16:00 [PATCH v12 3/4] Remove uses of popcount builtins. Nathan Bossart <nathan@postgresql.org>
2026-02-06 16:00 [PATCH v13 3/5] Remove uses of popcount builtins. Nathan Bossart <nathan@postgresql.org>
2026-02-06 16:00 [PATCH v14 1/2] Remove uses of popcount builtins. Nathan Bossart <nathan@postgresql.org>
2026-02-06 16:00 [PATCH v11 4/4] Remove uses of popcount builtins. Nathan Bossart <nathan@postgresql.org>
2026-02-06 16:00 [PATCH v14 1/2] Remove uses of popcount builtins. Nathan Bossart <nathan@postgresql.org>
2026-02-20 20:33 [PATCH v15 1/2] Remove uses of popcount builtins. Nathan Bossart <nathan@postgresql.org>
2026-02-21 21:12 [PATCH v16 1/2] Remove uses of popcount builtins. Nathan Bossart <nathan@postgresql.org>
2026-02-21 21:12 [PATCH v16 1/2] Remove uses of popcount builtins. Nathan Bossart <nathan@postgresql.org>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox