From: Nathan Bossart <nathandbossart@gmail.com>
To: John Naylor <johncnaylorls@gmail.com>
Cc: Heikki Linnakangas <hlinnaka@iki.fi>
Cc: pgsql-hackers@postgresql.org
Subject: Re: refactor architecture-specific popcount code
Date: Thu, 29 Jan 2026 11:06:03 -0600
Message-ID: <aXuTex5a4iu-BYXy@nathan> (raw)
In-Reply-To: <CANWCAZacr4AqrFzQRhhwQGwV1xSaUWyEYnYOT9SFc3cJFUWV6g@mail.gmail.com>
References: <e67ca6d9-db0b-4322-97ca-176964b94a35@iki.fi>
<aWkQ8AB9WlWVEEfe@nathan>
<652cab58-8dfb-4514-a6b0-218a3edb0699@iki.fi>
<aWk65qnA-wD0dOjp@nathan>
<CANWCAZaPMn0Xgy7nav_JxOLW-3RsZpd6c9Ad7KFCD4CrEirCkQ@mail.gmail.com>
<aXE117d0gWDwlPdA@nathan>
<CANWCAZY7R+iy+r9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ@mail.gmail.com>
<aXJjbkp2_glyfy6z@nathan>
<aXeLLydy1lxMwcCv@nathan>
<CANWCAZacr4AqrFzQRhhwQGwV1xSaUWyEYnYOT9SFc3cJFUWV6g@mail.gmail.com>
On Thu, Jan 29, 2026 at 06:31:53PM +0700, John Naylor wrote:
> On Mon, Jan 26, 2026 at 10:41 PM Nathan Bossart
> <nathandbossart@gmail.com> wrote:
>> I added a 0003 that swaps that final popcount with pg_popcount().
>
> I'm not sure either if this part matters much, but it makes more sense
> to me to continue using single word functions for that last part.
> Since they have very few call sites anymore, we can make them inline
> without bloating the binary on x86.
Okay, I abandoned that patch.
> Right, just the portable ones. Here, too, inlining them everywhere
> would mitigate any impact.
Done.
> +static inline int
> +pg_popcount64_neon(uint64 word)
>
> ...if they were inlined from the header, I think we wouldn't need this
> separate neon function in this file at all. Currently, we rely on
> __builtin_popcountl for the portable function outside this file. We
> could either keep using that or switch to neon if there's a
> portability difference.
Done.
--
nathan
From a85d39c3c73b621f195ad54ec12ef41f0ff3154c Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathan@postgresql.org>
Date: Thu, 22 Jan 2026 11:16:09 -0600
Subject: [PATCH v5 1/3] Make use of pg_popcount() in more places.
---
src/backend/nodes/bitmapset.c | 27 ++++-----------------------
src/include/lib/radixtree.h | 4 ++--
2 files changed, 6 insertions(+), 25 deletions(-)
diff --git a/src/backend/nodes/bitmapset.c b/src/backend/nodes/bitmapset.cindex a4765876c31..23c91fdb6c9 100644--- a/src/backend/nodes/bitmapset.c+++ b/src/backend/nodes/bitmapset.c@@ -553,14 +553,8 @@ bms_member_index(Bitmapset *a, int x)
bitnum = BITNUM(x);
/* count bits in preceding words */
- for (int i = 0; i < wordnum; i++)- {- bitmapword w = a->words[i];-- /* No need to count the bits in a zero word */- if (w != 0)- result += bmw_popcount(w);- }+ result += pg_popcount((const char *) a->words,+ wordnum * sizeof(bitmapword));
/*
* Now add bits of the last word, but only those before the item. We can
@@ -749,26 +743,13 @@ bms_get_singleton_member(const Bitmapset *a, int *member)
int
bms_num_members(const Bitmapset *a)
{
- int result = 0;- int nwords;- int wordnum;-
Assert(bms_is_valid_set(a));
if (a == NULL)
return 0;
- nwords = a->nwords;- wordnum = 0;- do- {- bitmapword w = a->words[wordnum];-- /* No need to count the bits in a zero word */- if (w != 0)- result += bmw_popcount(w);- } while (++wordnum < nwords);- return result;+ return pg_popcount((const char *) a->words,+ a->nwords * sizeof(bitmapword));
}
/*
diff --git a/src/include/lib/radixtree.h b/src/include/lib/radixtree.hindex b223ce10a2d..1425654a67c 100644--- a/src/include/lib/radixtree.h+++ b/src/include/lib/radixtree.h@@ -2725,8 +2725,8 @@ RT_VERIFY_NODE(RT_NODE * node)
/* RT_DUMP_NODE(node); */
- for (int i = 0; i < RT_BM_IDX(RT_NODE_MAX_SLOTS); i++)- cnt += bmw_popcount(n256->isset[i]);+ cnt += pg_popcount((const char *) n256->isset,+ RT_NODE_MAX_SLOTS / BITS_PER_BYTE);
/*
* Check if the number of used chunk matches, accounting for
--
2.50.1 (Apple Git-155)
From 959817e14a221463a95eaaddeb661b1b450d84e9 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathan@postgresql.org>
Date: Thu, 22 Jan 2026 11:33:56 -0600
Subject: [PATCH v5 2/3] Remove unnecessary 32-bit optimizations and alignment
checks.
---
src/port/pg_popcount_x86.c | 63 ++++++++------------------------------
1 file changed, 13 insertions(+), 50 deletions(-)
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.cindex 245f0167d00..0e98f532552 100644--- a/src/port/pg_popcount_x86.c+++ b/src/port/pg_popcount_x86.c@@ -382,33 +382,16 @@ pg_popcount_sse42(const char *buf, int bytes)
uint64 popcnt = 0;
#if SIZEOF_VOID_P >= 8
- /* Process in 64-bit chunks if the buffer is aligned. */- if (buf == (const char *) TYPEALIGN(8, buf))- {- const uint64 *words = (const uint64 *) buf;-- while (bytes >= 8)- {- popcnt += pg_popcount64_sse42(*words++);- bytes -= 8;- }+ /* Process in 64-bit chunks. */+ const uint64 *words = (const uint64 *) buf;- buf = (const char *) words;- }-#else- /* Process in 32-bit chunks if the buffer is aligned. */- if (buf == (const char *) TYPEALIGN(4, buf))+ while (bytes >= 8)
{
- const uint32 *words = (const uint32 *) buf;-- while (bytes >= 4)- {- popcnt += pg_popcount32_sse42(*words++);- bytes -= 4;- }-- buf = (const char *) words;+ popcnt += pg_popcount64_sse42(*words++);+ bytes -= 8;
}
++ buf = (const char *) words;
#endif
/* Process any remaining bytes */
@@ -428,37 +411,17 @@ pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask)
uint64 popcnt = 0;
#if SIZEOF_VOID_P >= 8
- /* Process in 64-bit chunks if the buffer is aligned */+ /* Process in 64-bit chunks. */
uint64 maskv = ~UINT64CONST(0) / 0xFF * mask;
+ const uint64 *words = (const uint64 *) buf;- if (buf == (const char *) TYPEALIGN(8, buf))+ while (bytes >= 8)
{
- const uint64 *words = (const uint64 *) buf;-- while (bytes >= 8)- {- popcnt += pg_popcount64_sse42(*words++ & maskv);- bytes -= 8;- }-- buf = (const char *) words;+ popcnt += pg_popcount64_sse42(*words++ & maskv);+ bytes -= 8;
}
-#else- /* Process in 32-bit chunks if the buffer is aligned. */- uint32 maskv = ~((uint32) 0) / 0xFF * mask;- if (buf == (const char *) TYPEALIGN(4, buf))- {- const uint32 *words = (const uint32 *) buf;-- while (bytes >= 4)- {- popcnt += pg_popcount32_sse42(*words++ & maskv);- bytes -= 4;- }-- buf = (const char *) words;- }+ buf = (const char *) words;
#endif
/* Process any remaining bytes */
--
2.50.1 (Apple Git-155)
From be044d31351a4adf3f69878d117146f87793a058 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathan@postgresql.org>
Date: Fri, 23 Jan 2026 17:31:20 -0600
Subject: [PATCH v5 3/3] Remove specialized word-length popcount
implementations.
---
src/include/port/pg_bitutils.h | 75 +++++++++++++++++++++++-----------
src/port/pg_bitutils.c | 69 ++-----------------------------
src/port/pg_popcount_aarch64.c | 25 ------------
src/port/pg_popcount_x86.c | 43 +------------------
4 files changed, 56 insertions(+), 156 deletions(-)
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.hindex 35761f509ec..a3e0f346ef6 100644--- a/src/include/port/pg_bitutils.h+++ b/src/include/port/pg_bitutils.h@@ -276,46 +276,73 @@ pg_ceil_log2_64(uint64 num)
return pg_leftmost_one_pos64(num - 1) + 1;
}
-extern int pg_popcount32_portable(uint32 word);-extern int pg_popcount64_portable(uint64 word);
extern uint64 pg_popcount_portable(const char *buf, int bytes);
extern uint64 pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask);
-#ifdef HAVE_X86_64_POPCNTQ+#if defined(HAVE_X86_64_POPCNTQ) || defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
/*
- * Attempt to use SSE4.2 or AVX-512 instructions, but perform a runtime check+ * Attempt to use specialized CPU instructions, but perform a runtime check
* first.
*/
-extern PGDLLIMPORT int (*pg_popcount32) (uint32 word);-extern PGDLLIMPORT int (*pg_popcount64) (uint64 word);
extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
-#elif defined(USE_NEON)-/* Use the Neon version of pg_popcount{32,64} without function pointer. */-extern int pg_popcount32(uint32 word);-extern int pg_popcount64(uint64 word);--/*- * We can try to use an SVE-optimized pg_popcount() on some systems For that,- * we do use a function pointer.- */-#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK-extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);-extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
#else
+/* Use a portable implementation -- no need for a function pointer. */
extern uint64 pg_popcount_optimized(const char *buf, int bytes);
extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+
#endif
-#else-/* Use a portable implementation -- no need for a function pointer. */-extern int pg_popcount32(uint32 word);-extern int pg_popcount64(uint64 word);-extern uint64 pg_popcount_optimized(const char *buf, int bytes);-extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);+/*+ * pg_popcount32+ * Return the number of 1 bits set in word+ */+static inline int+pg_popcount32(uint32 word)+{+#ifdef HAVE__BUILTIN_POPCOUNT+ return __builtin_popcount(word);+#else /* !HAVE__BUILTIN_POPCOUNT */+ int result = 0;++ while (word != 0)+ {+ result += pg_number_of_ones[word & 255];+ word >>= 8;+ }+ return result;+#endif /* HAVE__BUILTIN_POPCOUNT */+}++/*+ * pg_popcount64+ * Return the number of 1 bits set in word+ */+static inline int+pg_popcount64(uint64 word)+{+#ifdef HAVE__BUILTIN_POPCOUNT+#if SIZEOF_LONG == 8+ return __builtin_popcountl(word);+#elif SIZEOF_LONG_LONG == 8+ return __builtin_popcountll(word);+#else+#error "cannot find integer of the same size as uint64_t"
#endif
+#else /* !HAVE__BUILTIN_POPCOUNT */+ int result = 0;++ while (word != 0)+ {+ result += pg_number_of_ones[word & 255];+ word >>= 8;+ }++ return result;+#endif /* HAVE__BUILTIN_POPCOUNT */+}
/*
* Returns the number of 1-bits in buf.
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.cindex ffda75825e5..f942c1be55a 100644--- a/src/port/pg_bitutils.c+++ b/src/port/pg_bitutils.c@@ -96,56 +96,6 @@ const uint8 pg_number_of_ones[256] = {
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
};
-/*- * pg_popcount32_portable- * Return the number of 1 bits set in word- */-int-pg_popcount32_portable(uint32 word)-{-#ifdef HAVE__BUILTIN_POPCOUNT- return __builtin_popcount(word);-#else /* !HAVE__BUILTIN_POPCOUNT */- int result = 0;-- while (word != 0)- {- result += pg_number_of_ones[word & 255];- word >>= 8;- }-- return result;-#endif /* HAVE__BUILTIN_POPCOUNT */-}--/*- * pg_popcount64_portable- * Return the number of 1 bits set in word- */-int-pg_popcount64_portable(uint64 word)-{-#ifdef HAVE__BUILTIN_POPCOUNT-#if SIZEOF_LONG == 8- return __builtin_popcountl(word);-#elif SIZEOF_LONG_LONG == 8- return __builtin_popcountll(word);-#else-#error "cannot find integer of the same size as uint64_t"-#endif-#else /* !HAVE__BUILTIN_POPCOUNT */- int result = 0;-- while (word != 0)- {- result += pg_number_of_ones[word & 255];- word >>= 8;- }-- return result;-#endif /* HAVE__BUILTIN_POPCOUNT */-}-
/*
* pg_popcount_portable
* Returns the number of 1-bits in buf
@@ -163,7 +113,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++);+ popcnt += pg_popcount64(*words++);
bytes -= 8;
}
@@ -177,7 +127,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 4)
{
- popcnt += pg_popcount32_portable(*words++);+ popcnt += pg_popcount32(*words++);
bytes -= 4;
}
@@ -211,7 +161,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++ & maskv);+ popcnt += pg_popcount64(*words++ & maskv);
bytes -= 8;
}
@@ -227,7 +177,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 4)
{
- popcnt += pg_popcount32_portable(*words++ & maskv);+ popcnt += pg_popcount32(*words++ & maskv);
bytes -= 4;
}
@@ -250,17 +200,6 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
* actual external functions. The compiler should be able to inline the
* portable versions here.
*/
-int-pg_popcount32(uint32 word)-{- return pg_popcount32_portable(word);-}--int-pg_popcount64(uint64 word)-{- return pg_popcount64_portable(word);-}
/*
* pg_popcount_optimized
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.cindex ba57f2cd4bd..74f71593721 100644--- a/src/port/pg_popcount_aarch64.c+++ b/src/port/pg_popcount_aarch64.c@@ -291,31 +291,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
#endif /* ! USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
-/*- * pg_popcount32- * Return number of 1 bits in word- */-int-pg_popcount32(uint32 word)-{- return pg_popcount64((uint64) word);-}--/*- * pg_popcount64- * Return number of 1 bits in word- */-int-pg_popcount64(uint64 word)-{- /*- * For some compilers, __builtin_popcountl() already emits Neon- * instructions. The line below should compile to the same code on those- * systems.- */- return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));-}-
/*
* pg_popcount_neon
* Returns number of 1 bits in buf
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.cindex 0e98f532552..9fd8e18ed16 100644--- a/src/port/pg_popcount_x86.c+++ b/src/port/pg_popcount_x86.c@@ -36,8 +36,6 @@
* operation, but in practice this is close enough, and "sse42" seems easier to
* follow than "popcnt" for these names.
*/
-static inline int pg_popcount32_sse42(uint32 word);-static inline int pg_popcount64_sse42(uint64 word);
static uint64 pg_popcount_sse42(const char *buf, int bytes);
static uint64 pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask);
@@ -55,12 +53,8 @@ static uint64 pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask);
* what the current CPU supports) and then will call the pointer to fulfill the
* caller's request.
*/
-static int pg_popcount32_choose(uint32 word);-static int pg_popcount64_choose(uint64 word);
static uint64 pg_popcount_choose(const char *buf, int bytes);
static uint64 pg_popcount_masked_choose(const char *buf, int bytes, bits8 mask);
-int (*pg_popcount32) (uint32 word) = pg_popcount32_choose;-int (*pg_popcount64) (uint64 word) = pg_popcount64_choose;
uint64 (*pg_popcount_optimized) (const char *buf, int bytes) = pg_popcount_choose;
uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask) = pg_popcount_masked_choose;
@@ -157,7 +151,7 @@ pg_popcount_avx512_available(void)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
/*
- * These functions get called on the first call to pg_popcount32 etc.+ * These functions get called on the first call to pg_popcount(), etc.
* They detect whether we can use the asm implementations, and replace
* the function pointers so that subsequent calls are routed directly to
* the chosen implementation.
@@ -167,15 +161,11 @@ choose_popcount_functions(void)
{
if (pg_popcount_sse42_available())
{
- pg_popcount32 = pg_popcount32_sse42;- pg_popcount64 = pg_popcount64_sse42;
pg_popcount_optimized = pg_popcount_sse42;
pg_popcount_masked_optimized = pg_popcount_masked_sse42;
}
else
{
- pg_popcount32 = pg_popcount32_portable;- pg_popcount64 = pg_popcount64_portable;
pg_popcount_optimized = pg_popcount_portable;
pg_popcount_masked_optimized = pg_popcount_masked_portable;
}
@@ -189,20 +179,6 @@ choose_popcount_functions(void)
#endif
}
-static int-pg_popcount32_choose(uint32 word)-{- choose_popcount_functions();- return pg_popcount32(word);-}--static int-pg_popcount64_choose(uint64 word)-{- choose_popcount_functions();- return pg_popcount64(word);-}-
static uint64
pg_popcount_choose(const char *buf, int bytes)
{
@@ -338,23 +314,6 @@ pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
-/*- * pg_popcount32_sse42- * Return the number of 1 bits set in word- */-static inline int-pg_popcount32_sse42(uint32 word)-{-#ifdef _MSC_VER- return __popcnt(word);-#else- uint32 res;--__asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc");- return (int) res;-#endif-}-
/*
* pg_popcount64_sse42
* Return the number of 1 bits set in word
--
2.50.1 (Apple Git-155)
Attachments:
[text/plain] v5-0001-Make-use-of-pg_popcount-in-more-places.patch (2.1K, ../aXuTex5a4iu-BYXy@nathan/2-v5-0001-Make-use-of-pg_popcount-in-more-places.patch)
download | inline diff:
From a85d39c3c73b621f195ad54ec12ef41f0ff3154c Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathan@postgresql.org>
Date: Thu, 22 Jan 2026 11:16:09 -0600
Subject: [PATCH v5 1/3] Make use of pg_popcount() in more places.
---
src/backend/nodes/bitmapset.c | 27 ++++-----------------------
src/include/lib/radixtree.h | 4 ++--
2 files changed, 6 insertions(+), 25 deletions(-)
diff --git a/src/backend/nodes/bitmapset.c b/src/backend/nodes/bitmapset.cindex a4765876c31..23c91fdb6c9 100644--- a/src/backend/nodes/bitmapset.c+++ b/src/backend/nodes/bitmapset.c@@ -553,14 +553,8 @@ bms_member_index(Bitmapset *a, int x)
bitnum = BITNUM(x);
/* count bits in preceding words */
- for (int i = 0; i < wordnum; i++)- {- bitmapword w = a->words[i];-- /* No need to count the bits in a zero word */- if (w != 0)- result += bmw_popcount(w);- }+ result += pg_popcount((const char *) a->words,+ wordnum * sizeof(bitmapword));
/*
* Now add bits of the last word, but only those before the item. We can
@@ -749,26 +743,13 @@ bms_get_singleton_member(const Bitmapset *a, int *member)
int
bms_num_members(const Bitmapset *a)
{
- int result = 0;- int nwords;- int wordnum;-
Assert(bms_is_valid_set(a));
if (a == NULL)
return 0;
- nwords = a->nwords;- wordnum = 0;- do- {- bitmapword w = a->words[wordnum];-- /* No need to count the bits in a zero word */- if (w != 0)- result += bmw_popcount(w);- } while (++wordnum < nwords);- return result;+ return pg_popcount((const char *) a->words,+ a->nwords * sizeof(bitmapword));
}
/*
diff --git a/src/include/lib/radixtree.h b/src/include/lib/radixtree.hindex b223ce10a2d..1425654a67c 100644--- a/src/include/lib/radixtree.h+++ b/src/include/lib/radixtree.h@@ -2725,8 +2725,8 @@ RT_VERIFY_NODE(RT_NODE * node)
/* RT_DUMP_NODE(node); */
- for (int i = 0; i < RT_BM_IDX(RT_NODE_MAX_SLOTS); i++)- cnt += bmw_popcount(n256->isset[i]);+ cnt += pg_popcount((const char *) n256->isset,+ RT_NODE_MAX_SLOTS / BITS_PER_BYTE);
/*
* Check if the number of used chunk matches, accounting for
--
2.50.1 (Apple Git-155)
[text/plain] v5-0002-Remove-unnecessary-32-bit-optimizations-and-align.patch (2.5K, ../aXuTex5a4iu-BYXy@nathan/3-v5-0002-Remove-unnecessary-32-bit-optimizations-and-align.patch)
download | inline diff:
From 959817e14a221463a95eaaddeb661b1b450d84e9 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathan@postgresql.org>
Date: Thu, 22 Jan 2026 11:33:56 -0600
Subject: [PATCH v5 2/3] Remove unnecessary 32-bit optimizations and alignment
checks.
---
src/port/pg_popcount_x86.c | 63 ++++++++------------------------------
1 file changed, 13 insertions(+), 50 deletions(-)
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.cindex 245f0167d00..0e98f532552 100644--- a/src/port/pg_popcount_x86.c+++ b/src/port/pg_popcount_x86.c@@ -382,33 +382,16 @@ pg_popcount_sse42(const char *buf, int bytes)
uint64 popcnt = 0;
#if SIZEOF_VOID_P >= 8
- /* Process in 64-bit chunks if the buffer is aligned. */- if (buf == (const char *) TYPEALIGN(8, buf))- {- const uint64 *words = (const uint64 *) buf;-- while (bytes >= 8)- {- popcnt += pg_popcount64_sse42(*words++);- bytes -= 8;- }+ /* Process in 64-bit chunks. */+ const uint64 *words = (const uint64 *) buf;- buf = (const char *) words;- }-#else- /* Process in 32-bit chunks if the buffer is aligned. */- if (buf == (const char *) TYPEALIGN(4, buf))+ while (bytes >= 8)
{
- const uint32 *words = (const uint32 *) buf;-- while (bytes >= 4)- {- popcnt += pg_popcount32_sse42(*words++);- bytes -= 4;- }-- buf = (const char *) words;+ popcnt += pg_popcount64_sse42(*words++);+ bytes -= 8;
}
++ buf = (const char *) words;
#endif
/* Process any remaining bytes */
@@ -428,37 +411,17 @@ pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask)
uint64 popcnt = 0;
#if SIZEOF_VOID_P >= 8
- /* Process in 64-bit chunks if the buffer is aligned */+ /* Process in 64-bit chunks. */
uint64 maskv = ~UINT64CONST(0) / 0xFF * mask;
+ const uint64 *words = (const uint64 *) buf;- if (buf == (const char *) TYPEALIGN(8, buf))+ while (bytes >= 8)
{
- const uint64 *words = (const uint64 *) buf;-- while (bytes >= 8)- {- popcnt += pg_popcount64_sse42(*words++ & maskv);- bytes -= 8;- }-- buf = (const char *) words;+ popcnt += pg_popcount64_sse42(*words++ & maskv);+ bytes -= 8;
}
-#else- /* Process in 32-bit chunks if the buffer is aligned. */- uint32 maskv = ~((uint32) 0) / 0xFF * mask;- if (buf == (const char *) TYPEALIGN(4, buf))- {- const uint32 *words = (const uint32 *) buf;-- while (bytes >= 4)- {- popcnt += pg_popcount32_sse42(*words++ & maskv);- bytes -= 4;- }-- buf = (const char *) words;- }+ buf = (const char *) words;
#endif
/* Process any remaining bytes */
--
2.50.1 (Apple Git-155)
[text/plain] v5-0003-Remove-specialized-word-length-popcount-implement.patch (10.2K, ../aXuTex5a4iu-BYXy@nathan/4-v5-0003-Remove-specialized-word-length-popcount-implement.patch)
download | inline diff:
From be044d31351a4adf3f69878d117146f87793a058 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathan@postgresql.org>
Date: Fri, 23 Jan 2026 17:31:20 -0600
Subject: [PATCH v5 3/3] Remove specialized word-length popcount
implementations.
---
src/include/port/pg_bitutils.h | 75 +++++++++++++++++++++++-----------
src/port/pg_bitutils.c | 69 ++-----------------------------
src/port/pg_popcount_aarch64.c | 25 ------------
src/port/pg_popcount_x86.c | 43 +------------------
4 files changed, 56 insertions(+), 156 deletions(-)
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.hindex 35761f509ec..a3e0f346ef6 100644--- a/src/include/port/pg_bitutils.h+++ b/src/include/port/pg_bitutils.h@@ -276,46 +276,73 @@ pg_ceil_log2_64(uint64 num)
return pg_leftmost_one_pos64(num - 1) + 1;
}
-extern int pg_popcount32_portable(uint32 word);-extern int pg_popcount64_portable(uint64 word);
extern uint64 pg_popcount_portable(const char *buf, int bytes);
extern uint64 pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask);
-#ifdef HAVE_X86_64_POPCNTQ+#if defined(HAVE_X86_64_POPCNTQ) || defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
/*
- * Attempt to use SSE4.2 or AVX-512 instructions, but perform a runtime check+ * Attempt to use specialized CPU instructions, but perform a runtime check
* first.
*/
-extern PGDLLIMPORT int (*pg_popcount32) (uint32 word);-extern PGDLLIMPORT int (*pg_popcount64) (uint64 word);
extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
-#elif defined(USE_NEON)-/* Use the Neon version of pg_popcount{32,64} without function pointer. */-extern int pg_popcount32(uint32 word);-extern int pg_popcount64(uint64 word);--/*- * We can try to use an SVE-optimized pg_popcount() on some systems For that,- * we do use a function pointer.- */-#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK-extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);-extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
#else
+/* Use a portable implementation -- no need for a function pointer. */
extern uint64 pg_popcount_optimized(const char *buf, int bytes);
extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+
#endif
-#else-/* Use a portable implementation -- no need for a function pointer. */-extern int pg_popcount32(uint32 word);-extern int pg_popcount64(uint64 word);-extern uint64 pg_popcount_optimized(const char *buf, int bytes);-extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);+/*+ * pg_popcount32+ * Return the number of 1 bits set in word+ */+static inline int+pg_popcount32(uint32 word)+{+#ifdef HAVE__BUILTIN_POPCOUNT+ return __builtin_popcount(word);+#else /* !HAVE__BUILTIN_POPCOUNT */+ int result = 0;++ while (word != 0)+ {+ result += pg_number_of_ones[word & 255];+ word >>= 8;+ }+ return result;+#endif /* HAVE__BUILTIN_POPCOUNT */+}++/*+ * pg_popcount64+ * Return the number of 1 bits set in word+ */+static inline int+pg_popcount64(uint64 word)+{+#ifdef HAVE__BUILTIN_POPCOUNT+#if SIZEOF_LONG == 8+ return __builtin_popcountl(word);+#elif SIZEOF_LONG_LONG == 8+ return __builtin_popcountll(word);+#else+#error "cannot find integer of the same size as uint64_t"
#endif
+#else /* !HAVE__BUILTIN_POPCOUNT */+ int result = 0;++ while (word != 0)+ {+ result += pg_number_of_ones[word & 255];+ word >>= 8;+ }++ return result;+#endif /* HAVE__BUILTIN_POPCOUNT */+}
/*
* Returns the number of 1-bits in buf.
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.cindex ffda75825e5..f942c1be55a 100644--- a/src/port/pg_bitutils.c+++ b/src/port/pg_bitutils.c@@ -96,56 +96,6 @@ const uint8 pg_number_of_ones[256] = {
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
};
-/*- * pg_popcount32_portable- * Return the number of 1 bits set in word- */-int-pg_popcount32_portable(uint32 word)-{-#ifdef HAVE__BUILTIN_POPCOUNT- return __builtin_popcount(word);-#else /* !HAVE__BUILTIN_POPCOUNT */- int result = 0;-- while (word != 0)- {- result += pg_number_of_ones[word & 255];- word >>= 8;- }-- return result;-#endif /* HAVE__BUILTIN_POPCOUNT */-}--/*- * pg_popcount64_portable- * Return the number of 1 bits set in word- */-int-pg_popcount64_portable(uint64 word)-{-#ifdef HAVE__BUILTIN_POPCOUNT-#if SIZEOF_LONG == 8- return __builtin_popcountl(word);-#elif SIZEOF_LONG_LONG == 8- return __builtin_popcountll(word);-#else-#error "cannot find integer of the same size as uint64_t"-#endif-#else /* !HAVE__BUILTIN_POPCOUNT */- int result = 0;-- while (word != 0)- {- result += pg_number_of_ones[word & 255];- word >>= 8;- }-- return result;-#endif /* HAVE__BUILTIN_POPCOUNT */-}-
/*
* pg_popcount_portable
* Returns the number of 1-bits in buf
@@ -163,7 +113,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++);+ popcnt += pg_popcount64(*words++);
bytes -= 8;
}
@@ -177,7 +127,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 4)
{
- popcnt += pg_popcount32_portable(*words++);+ popcnt += pg_popcount32(*words++);
bytes -= 4;
}
@@ -211,7 +161,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++ & maskv);+ popcnt += pg_popcount64(*words++ & maskv);
bytes -= 8;
}
@@ -227,7 +177,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 4)
{
- popcnt += pg_popcount32_portable(*words++ & maskv);+ popcnt += pg_popcount32(*words++ & maskv);
bytes -= 4;
}
@@ -250,17 +200,6 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
* actual external functions. The compiler should be able to inline the
* portable versions here.
*/
-int-pg_popcount32(uint32 word)-{- return pg_popcount32_portable(word);-}--int-pg_popcount64(uint64 word)-{- return pg_popcount64_portable(word);-}
/*
* pg_popcount_optimized
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.cindex ba57f2cd4bd..74f71593721 100644--- a/src/port/pg_popcount_aarch64.c+++ b/src/port/pg_popcount_aarch64.c@@ -291,31 +291,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
#endif /* ! USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
-/*- * pg_popcount32- * Return number of 1 bits in word- */-int-pg_popcount32(uint32 word)-{- return pg_popcount64((uint64) word);-}--/*- * pg_popcount64- * Return number of 1 bits in word- */-int-pg_popcount64(uint64 word)-{- /*- * For some compilers, __builtin_popcountl() already emits Neon- * instructions. The line below should compile to the same code on those- * systems.- */- return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));-}-
/*
* pg_popcount_neon
* Returns number of 1 bits in buf
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.cindex 0e98f532552..9fd8e18ed16 100644--- a/src/port/pg_popcount_x86.c+++ b/src/port/pg_popcount_x86.c@@ -36,8 +36,6 @@
* operation, but in practice this is close enough, and "sse42" seems easier to
* follow than "popcnt" for these names.
*/
-static inline int pg_popcount32_sse42(uint32 word);-static inline int pg_popcount64_sse42(uint64 word);
static uint64 pg_popcount_sse42(const char *buf, int bytes);
static uint64 pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask);
@@ -55,12 +53,8 @@ static uint64 pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask);
* what the current CPU supports) and then will call the pointer to fulfill the
* caller's request.
*/
-static int pg_popcount32_choose(uint32 word);-static int pg_popcount64_choose(uint64 word);
static uint64 pg_popcount_choose(const char *buf, int bytes);
static uint64 pg_popcount_masked_choose(const char *buf, int bytes, bits8 mask);
-int (*pg_popcount32) (uint32 word) = pg_popcount32_choose;-int (*pg_popcount64) (uint64 word) = pg_popcount64_choose;
uint64 (*pg_popcount_optimized) (const char *buf, int bytes) = pg_popcount_choose;
uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask) = pg_popcount_masked_choose;
@@ -157,7 +151,7 @@ pg_popcount_avx512_available(void)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
/*
- * These functions get called on the first call to pg_popcount32 etc.+ * These functions get called on the first call to pg_popcount(), etc.
* They detect whether we can use the asm implementations, and replace
* the function pointers so that subsequent calls are routed directly to
* the chosen implementation.
@@ -167,15 +161,11 @@ choose_popcount_functions(void)
{
if (pg_popcount_sse42_available())
{
- pg_popcount32 = pg_popcount32_sse42;- pg_popcount64 = pg_popcount64_sse42;
pg_popcount_optimized = pg_popcount_sse42;
pg_popcount_masked_optimized = pg_popcount_masked_sse42;
}
else
{
- pg_popcount32 = pg_popcount32_portable;- pg_popcount64 = pg_popcount64_portable;
pg_popcount_optimized = pg_popcount_portable;
pg_popcount_masked_optimized = pg_popcount_masked_portable;
}
@@ -189,20 +179,6 @@ choose_popcount_functions(void)
#endif
}
-static int-pg_popcount32_choose(uint32 word)-{- choose_popcount_functions();- return pg_popcount32(word);-}--static int-pg_popcount64_choose(uint64 word)-{- choose_popcount_functions();- return pg_popcount64(word);-}-
static uint64
pg_popcount_choose(const char *buf, int bytes)
{
@@ -338,23 +314,6 @@ pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
-/*- * pg_popcount32_sse42- * Return the number of 1 bits set in word- */-static inline int-pg_popcount32_sse42(uint32 word)-{-#ifdef _MSC_VER- return __popcnt(word);-#else- uint32 res;--__asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc");- return (int) res;-#endif-}-
/*
* pg_popcount64_sse42
* Return the number of 1 bits set in word
--
2.50.1 (Apple Git-155)
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-hackers@postgresql.org
Cc: nathandbossart@gmail.com, johncnaylorls@gmail.com, hlinnaka@iki.fi
Subject: Re: refactor architecture-specific popcount code
In-Reply-To: <aXuTex5a4iu-BYXy@nathan>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox