agora inbox for pgsql-hackers@postgresql.org
help / color / mirror / Atom feed[PATCH 1/5] Make XLogReaderInvalReadState static.
19+ messages / 7 participants
[nested] [flat]
* [PATCH 1/5] Make XLogReaderInvalReadState static.
@ 2019-07-09 09:54 Antonin Houska <ah@cybertec.at>
0 siblings, 0 replies; 19+ messages in thread
From: Antonin Houska @ 2019-07-09 09:54 UTC (permalink / raw)
This change is not necessary for the XLogRead() refactoring itself, but I
noticed the problem while working on it. Not sure it's worth a separate CF
entry.
---
src/backend/access/transam/xlogreader.c | 4 ++--
src/include/access/xlogreader.h | 4 ----
2 files changed, 2 insertions(+), 6 deletions(-)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 41dae916b4..08bc9695c1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -35,10 +35,10 @@ static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
+static void XLogReaderInvalReadState(XLogReaderState *state);
static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
int reqLen);
static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3);
-
static void ResetDecoder(XLogReaderState *state);
/* size of the buffer allocated for error message. */
@@ -620,7 +620,7 @@ err:
/*
* Invalidate the xlogreader's read state to force a re-read.
*/
-void
+static void
XLogReaderInvalReadState(XLogReaderState *state)
{
state->readSegNo = 0;
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 04228e2a87..a3d3cc1e7b 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -212,13 +212,9 @@ extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
XLogRecPtr recptr, char *phdr);
-/* Invalidate read state */
-extern void XLogReaderInvalReadState(XLogReaderState *state);
-
#ifdef FRONTEND
extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
#endif /* FRONTEND */
-
/* Functions for decoding an XLogRecord */
extern bool DecodeXLogRecord(XLogReaderState *state, XLogRecord *record,
--
2.16.4
--=-=-=
Content-Type: text/x-diff
Content-Disposition: attachment;
filename=v04-0002-Remove-TLI-from-some-argument-lists.patch
^ permalink raw reply [nested|flat] 19+ messages in thread
* [PATCH v3] psql: Add tab-complete for optional view parameters
@ 2023-08-07 18:37 Christoph Heiss <christoph@c8h4.io>
0 siblings, 0 replies; 19+ messages in thread
From: Christoph Heiss @ 2023-08-07 18:37 UTC (permalink / raw)
This adds them in the same matter as it works for storage parameters of
tables.
Signed-off-by: Christoph Heiss <christoph@c8h4.io>
---
src/bin/psql/tab-complete.c | 36 +++++++++++++++++++++++++++++++-----
1 file changed, 31 insertions(+), 5 deletions(-)
diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c
index 779fdc90cb..83ec1508bb 100644
--- a/src/bin/psql/tab-complete.c
+++ b/src/bin/psql/tab-complete.c
@@ -1329,6 +1329,13 @@ static const char *const table_storage_parameters[] = {
NULL
};
+/* Optional parameters for CREATE VIEW and ALTER VIEW */
+static const char *const view_optional_parameters[] = {
+ "check_option",
+ "security_barrier",
+ "security_invoker",
+ NULL
+};
/* Forward declaration of functions */
static char **psql_completion(const char *text, int start, int end);
@@ -2216,8 +2223,7 @@ psql_completion(const char *text, int start, int end)
COMPLETE_WITH("TO");
/* ALTER VIEW <name> */
else if (Matches("ALTER", "VIEW", MatchAny))
- COMPLETE_WITH("ALTER COLUMN", "OWNER TO", "RENAME",
- "SET SCHEMA");
+ COMPLETE_WITH("ALTER COLUMN", "OWNER TO", "RENAME", "RESET (", "SET");
/* ALTER VIEW xxx RENAME */
else if (Matches("ALTER", "VIEW", MatchAny, "RENAME"))
COMPLETE_WITH_ATTR_PLUS(prev2_wd, "COLUMN", "TO");
@@ -2233,6 +2239,16 @@ psql_completion(const char *text, int start, int end)
/* ALTER VIEW xxx RENAME COLUMN yyy */
else if (Matches("ALTER", "VIEW", MatchAny, "RENAME", "COLUMN", MatchAnyExcept("TO")))
COMPLETE_WITH("TO");
+ /* ALTER VIEW xxx SET ( yyy [= zzz] ) */
+ else if (Matches("ALTER", "VIEW", MatchAny, "SET"))
+ COMPLETE_WITH("(", "SCHEMA");
+ /* ALTER VIEW xxx SET|RESET ( yyy [= zzz] ) */
+ else if (Matches("ALTER", "VIEW", MatchAny, "SET|RESET", "("))
+ COMPLETE_WITH_LIST(view_optional_parameters);
+ else if (Matches("ALTER", "VIEW", MatchAny, "SET", "(", "check_option", "="))
+ COMPLETE_WITH("local", "cascaded");
+ else if (Matches("ALTER", "VIEW", MatchAny, "SET", "(", "security_barrier|security_invoker", "="))
+ COMPLETE_WITH("true", "false");
/* ALTER MATERIALIZED VIEW <name> */
else if (Matches("ALTER", "MATERIALIZED", "VIEW", MatchAny))
@@ -3525,13 +3541,23 @@ psql_completion(const char *text, int start, int end)
}
/* CREATE VIEW --- is allowed inside CREATE SCHEMA, so use TailMatches */
- /* Complete CREATE [ OR REPLACE ] VIEW <name> with AS */
+ /* Complete CREATE [ OR REPLACE ] VIEW <name> with AS or WITH ( */
else if (TailMatches("CREATE", "VIEW", MatchAny) ||
TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny))
+ COMPLETE_WITH("AS", "WITH (");
+ /* Complete CREATE [ OR REPLACE ] VIEW <name> WITH ( with supported options */
+ else if (TailMatches("CREATE", "VIEW", MatchAny, "WITH", "(") ||
+ TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "WITH", "("))
+ COMPLETE_WITH_LIST(view_optional_parameters);
+ /* Complete CREATE [ OR REPLACE ] VIEW <name> WITH ( ... ) with "AS" */
+ else if (TailMatches("CREATE", "VIEW", MatchAny, "WITH", "(*)") ||
+ TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "WITH", "(*)"))
COMPLETE_WITH("AS");
- /* Complete "CREATE [ OR REPLACE ] VIEW <sth> AS with "SELECT" */
+ /* Complete "CREATE [ OR REPLACE ] VIEW <sth> [ WITH ( ... ) ] AS with "SELECT" */
else if (TailMatches("CREATE", "VIEW", MatchAny, "AS") ||
- TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "AS"))
+ TailMatches("CREATE", "VIEW", MatchAny, "WITH", "(*)", "AS") ||
+ TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "AS") ||
+ TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "WITH", "(*)", "AS"))
COMPLETE_WITH("SELECT");
/* CREATE MATERIALIZED VIEW */
--
2.41.0
--h2n4c22ub7j2vbdx--
^ permalink raw reply [nested|flat] 19+ messages in thread
* [PATCH v3] psql: Add tab-complete for optional view parameters
@ 2023-08-07 18:37 Christoph Heiss <christoph@c8h4.io>
0 siblings, 0 replies; 19+ messages in thread
From: Christoph Heiss @ 2023-08-07 18:37 UTC (permalink / raw)
This adds them in the same matter as it works for storage parameters of
tables.
Signed-off-by: Christoph Heiss <christoph@c8h4.io>
---
src/bin/psql/tab-complete.c | 36 +++++++++++++++++++++++++++++++-----
1 file changed, 31 insertions(+), 5 deletions(-)
diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c
index 779fdc90cb..83ec1508bb 100644
--- a/src/bin/psql/tab-complete.c
+++ b/src/bin/psql/tab-complete.c
@@ -1329,6 +1329,13 @@ static const char *const table_storage_parameters[] = {
NULL
};
+/* Optional parameters for CREATE VIEW and ALTER VIEW */
+static const char *const view_optional_parameters[] = {
+ "check_option",
+ "security_barrier",
+ "security_invoker",
+ NULL
+};
/* Forward declaration of functions */
static char **psql_completion(const char *text, int start, int end);
@@ -2216,8 +2223,7 @@ psql_completion(const char *text, int start, int end)
COMPLETE_WITH("TO");
/* ALTER VIEW <name> */
else if (Matches("ALTER", "VIEW", MatchAny))
- COMPLETE_WITH("ALTER COLUMN", "OWNER TO", "RENAME",
- "SET SCHEMA");
+ COMPLETE_WITH("ALTER COLUMN", "OWNER TO", "RENAME", "RESET (", "SET");
/* ALTER VIEW xxx RENAME */
else if (Matches("ALTER", "VIEW", MatchAny, "RENAME"))
COMPLETE_WITH_ATTR_PLUS(prev2_wd, "COLUMN", "TO");
@@ -2233,6 +2239,16 @@ psql_completion(const char *text, int start, int end)
/* ALTER VIEW xxx RENAME COLUMN yyy */
else if (Matches("ALTER", "VIEW", MatchAny, "RENAME", "COLUMN", MatchAnyExcept("TO")))
COMPLETE_WITH("TO");
+ /* ALTER VIEW xxx SET ( yyy [= zzz] ) */
+ else if (Matches("ALTER", "VIEW", MatchAny, "SET"))
+ COMPLETE_WITH("(", "SCHEMA");
+ /* ALTER VIEW xxx SET|RESET ( yyy [= zzz] ) */
+ else if (Matches("ALTER", "VIEW", MatchAny, "SET|RESET", "("))
+ COMPLETE_WITH_LIST(view_optional_parameters);
+ else if (Matches("ALTER", "VIEW", MatchAny, "SET", "(", "check_option", "="))
+ COMPLETE_WITH("local", "cascaded");
+ else if (Matches("ALTER", "VIEW", MatchAny, "SET", "(", "security_barrier|security_invoker", "="))
+ COMPLETE_WITH("true", "false");
/* ALTER MATERIALIZED VIEW <name> */
else if (Matches("ALTER", "MATERIALIZED", "VIEW", MatchAny))
@@ -3525,13 +3541,23 @@ psql_completion(const char *text, int start, int end)
}
/* CREATE VIEW --- is allowed inside CREATE SCHEMA, so use TailMatches */
- /* Complete CREATE [ OR REPLACE ] VIEW <name> with AS */
+ /* Complete CREATE [ OR REPLACE ] VIEW <name> with AS or WITH ( */
else if (TailMatches("CREATE", "VIEW", MatchAny) ||
TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny))
+ COMPLETE_WITH("AS", "WITH (");
+ /* Complete CREATE [ OR REPLACE ] VIEW <name> WITH ( with supported options */
+ else if (TailMatches("CREATE", "VIEW", MatchAny, "WITH", "(") ||
+ TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "WITH", "("))
+ COMPLETE_WITH_LIST(view_optional_parameters);
+ /* Complete CREATE [ OR REPLACE ] VIEW <name> WITH ( ... ) with "AS" */
+ else if (TailMatches("CREATE", "VIEW", MatchAny, "WITH", "(*)") ||
+ TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "WITH", "(*)"))
COMPLETE_WITH("AS");
- /* Complete "CREATE [ OR REPLACE ] VIEW <sth> AS with "SELECT" */
+ /* Complete "CREATE [ OR REPLACE ] VIEW <sth> [ WITH ( ... ) ] AS with "SELECT" */
else if (TailMatches("CREATE", "VIEW", MatchAny, "AS") ||
- TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "AS"))
+ TailMatches("CREATE", "VIEW", MatchAny, "WITH", "(*)", "AS") ||
+ TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "AS") ||
+ TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "WITH", "(*)", "AS"))
COMPLETE_WITH("SELECT");
/* CREATE MATERIALIZED VIEW */
--
2.41.0
--h2n4c22ub7j2vbdx--
^ permalink raw reply [nested|flat] 19+ messages in thread
* [PATCH v3] psql: Add tab-complete for optional view parameters
@ 2023-08-07 18:37 Christoph Heiss <christoph@c8h4.io>
0 siblings, 0 replies; 19+ messages in thread
From: Christoph Heiss @ 2023-08-07 18:37 UTC (permalink / raw)
This adds them in the same matter as it works for storage parameters of
tables.
Signed-off-by: Christoph Heiss <christoph@c8h4.io>
---
src/bin/psql/tab-complete.c | 36 +++++++++++++++++++++++++++++++-----
1 file changed, 31 insertions(+), 5 deletions(-)
diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c
index 779fdc90cb..83ec1508bb 100644
--- a/src/bin/psql/tab-complete.c
+++ b/src/bin/psql/tab-complete.c
@@ -1329,6 +1329,13 @@ static const char *const table_storage_parameters[] = {
NULL
};
+/* Optional parameters for CREATE VIEW and ALTER VIEW */
+static const char *const view_optional_parameters[] = {
+ "check_option",
+ "security_barrier",
+ "security_invoker",
+ NULL
+};
/* Forward declaration of functions */
static char **psql_completion(const char *text, int start, int end);
@@ -2216,8 +2223,7 @@ psql_completion(const char *text, int start, int end)
COMPLETE_WITH("TO");
/* ALTER VIEW <name> */
else if (Matches("ALTER", "VIEW", MatchAny))
- COMPLETE_WITH("ALTER COLUMN", "OWNER TO", "RENAME",
- "SET SCHEMA");
+ COMPLETE_WITH("ALTER COLUMN", "OWNER TO", "RENAME", "RESET (", "SET");
/* ALTER VIEW xxx RENAME */
else if (Matches("ALTER", "VIEW", MatchAny, "RENAME"))
COMPLETE_WITH_ATTR_PLUS(prev2_wd, "COLUMN", "TO");
@@ -2233,6 +2239,16 @@ psql_completion(const char *text, int start, int end)
/* ALTER VIEW xxx RENAME COLUMN yyy */
else if (Matches("ALTER", "VIEW", MatchAny, "RENAME", "COLUMN", MatchAnyExcept("TO")))
COMPLETE_WITH("TO");
+ /* ALTER VIEW xxx SET ( yyy [= zzz] ) */
+ else if (Matches("ALTER", "VIEW", MatchAny, "SET"))
+ COMPLETE_WITH("(", "SCHEMA");
+ /* ALTER VIEW xxx SET|RESET ( yyy [= zzz] ) */
+ else if (Matches("ALTER", "VIEW", MatchAny, "SET|RESET", "("))
+ COMPLETE_WITH_LIST(view_optional_parameters);
+ else if (Matches("ALTER", "VIEW", MatchAny, "SET", "(", "check_option", "="))
+ COMPLETE_WITH("local", "cascaded");
+ else if (Matches("ALTER", "VIEW", MatchAny, "SET", "(", "security_barrier|security_invoker", "="))
+ COMPLETE_WITH("true", "false");
/* ALTER MATERIALIZED VIEW <name> */
else if (Matches("ALTER", "MATERIALIZED", "VIEW", MatchAny))
@@ -3525,13 +3541,23 @@ psql_completion(const char *text, int start, int end)
}
/* CREATE VIEW --- is allowed inside CREATE SCHEMA, so use TailMatches */
- /* Complete CREATE [ OR REPLACE ] VIEW <name> with AS */
+ /* Complete CREATE [ OR REPLACE ] VIEW <name> with AS or WITH ( */
else if (TailMatches("CREATE", "VIEW", MatchAny) ||
TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny))
+ COMPLETE_WITH("AS", "WITH (");
+ /* Complete CREATE [ OR REPLACE ] VIEW <name> WITH ( with supported options */
+ else if (TailMatches("CREATE", "VIEW", MatchAny, "WITH", "(") ||
+ TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "WITH", "("))
+ COMPLETE_WITH_LIST(view_optional_parameters);
+ /* Complete CREATE [ OR REPLACE ] VIEW <name> WITH ( ... ) with "AS" */
+ else if (TailMatches("CREATE", "VIEW", MatchAny, "WITH", "(*)") ||
+ TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "WITH", "(*)"))
COMPLETE_WITH("AS");
- /* Complete "CREATE [ OR REPLACE ] VIEW <sth> AS with "SELECT" */
+ /* Complete "CREATE [ OR REPLACE ] VIEW <sth> [ WITH ( ... ) ] AS with "SELECT" */
else if (TailMatches("CREATE", "VIEW", MatchAny, "AS") ||
- TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "AS"))
+ TailMatches("CREATE", "VIEW", MatchAny, "WITH", "(*)", "AS") ||
+ TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "AS") ||
+ TailMatches("CREATE", "OR", "REPLACE", "VIEW", MatchAny, "WITH", "(*)", "AS"))
COMPLETE_WITH("SELECT");
/* CREATE MATERIALIZED VIEW */
--
2.41.0
--h2n4c22ub7j2vbdx--
^ permalink raw reply [nested|flat] 19+ messages in thread
* [PATCH v4 2/8] Address space reservation for shared memory
@ 2024-10-16 18:21 Dmitrii Dolgov <9erthalion6@gmail.com>
0 siblings, 0 replies; 19+ messages in thread
From: Dmitrii Dolgov @ 2024-10-16 18:21 UTC (permalink / raw)
Currently the kernel is responsible to chose an address, where to place each
shared memory mapping, which is the lowest possible address that do not clash
with any other mappings. This is considered to be the most portable approach,
but one of the downsides is that there is no place to resize allocated mappings
anymore. Here is how it looks like for one mapping in /proc/$PID/maps,
/dev/zero represents the anonymous shared memory we talk about:
00400000-00490000 /path/bin/postgres
...
012d9000-0133e000 [heap]
7f443a800000-7f470a800000 /dev/zero (deleted)
7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
...
7f471aef2000-7f471aef9000 /dev/shm/PostgreSQL.3859891842
7f471aef9000-7f471aefa000 /SYSV007dbf7d (deleted)
By specifying the mapping address directly it's possible to place the
mapping in a way that leaves room for resizing. The idea is:
* To reserve some address space via mmap'ing a large chunk of memory
with PROT_NONE and MAP_NORESERVE. This way we prepare a playground for
preparing shared memory layout without risking anything interfering
with that.
* To slice the reserved space up into sections, one to use for each
shared segment.
* Allocate shared memory segments out of corresponding slices and
leaving unclaimed space in between them. This is implemented via
mmap'ing memory at a specified address from the reserved space with
MAP_FIXED.
The result looks like this:
012d9000-0133e000 [heap]
7f443a800000-7f444196c000 /dev/zero (deleted)
7f444196c000-7f470a800000 # reserved space
7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
Things like address space randomization should not be a problem in this
context, since the randomization is applied to the mmap base, which is
one per process.
This approach also do not impact the actual memory usage as reported by
the kernel. Here is the output of /proc/$PID/status for the master
version with shared_buffers = 128 MB:
// Peak virtual memory size, which is described as total pages
// mapped in mm_struct. It corresponds to the mapped reserved space
// and is the only number that grows with it.
VmPeak: 2043192 kB
// Size of memory portions. It contains RssAnon + RssFile + RssShmem
VmRSS: 22908 kB
// Size of resident anonymous memory
RssAnon: 768 kB
// Size of resident file mappings
RssFile: 10364 kB
// Size of resident shmem memory (includes SysV shm, mapping of tmpfs and
// shared anonymous mappings)
RssShmem: 11776 kB
Here is the same for the patch when reserving 20GB of space:
VmPeak: 21250648 kB
VmRSS: 22948 kB
RssAnon: 768 kB
RssFile: 10404 kB
RssShmem: 11776 kB
Cgroup v2 doesn't have any problems with that as well. To verify a new cgroup
was created with the memory limit 256 MB, then PostgreSQL was launched withing
this cgroup with shared_buffers = 128 MB:
$ cd /sys/fs/cgroup
$ mkdir postgres
$ cd postres
$ echo 268435456 > memory.max
$ echo $MASTER_PID_SHELL > cgroup.procs
# postgres from the master branch has being successfully launched
# from that shell
$ cat memory.current
17465344 (~16.6 MB)
# stop postgres
$ echo $PATCH_PID_SHELL > cgroup.procs
# postgres from the patch has being successfully launched from that shell
$ cat memory.current
17637376 (~16.8 MB)
To control the amount of space reserved a new GUC max_available_memory
is introduced. Ideally it should be based on the maximum available
memory, hense the name.
---
src/backend/port/sysv_shmem.c | 284 ++++++++++++++++++++++++----
src/backend/port/win32_shmem.c | 2 +-
src/backend/storage/ipc/ipci.c | 5 +-
src/backend/utils/init/globals.c | 1 +
src/backend/utils/misc/guc_tables.c | 14 ++
src/include/storage/pg_shmem.h | 4 +-
6 files changed, 271 insertions(+), 39 deletions(-)
diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 56af0231d24..a0f03ff868f 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -108,6 +108,66 @@ static AnonymousMapping Mappings[ANON_MAPPINGS];
/* Keeps track of used mapping segments */
static int next_free_segment = 0;
+/*
+ * Anonymous mapping placing (/dev/zero (deleted) below) looks like this:
+ *
+ * 00400000-00490000 /path/bin/postgres
+ * ...
+ * 012d9000-0133e000 [heap]
+ * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
+ * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
+ * ...
+ * 7f471aef2000-7f471aef9000 /dev/shm/PostgreSQL.3859891842
+ * 7f471aef9000-7f471aefa000 /SYSV007dbf7d (deleted)
+ * ...
+ *
+ * We would like to place multiple mappings in such a way, that there will be
+ * enough space between them in the address space to be able to resize up to
+ * certain size, but without counting towards the total memory consumption.
+ *
+ * To achieve that we first reserve some shared memory address space by
+ * mmap'ing a segment of MaxAvailableMemory size with PROT_NONE and
+ * MAP_NORESERVE (these flags allow to make sure this space will not be used by
+ * anything else, yet do not count against memory limits). Having the reserved
+ * space, we allocate out of it actual chunks of shared memory as usual,
+ * updating a pointer to the current available reserved space for the next
+ * allocation with the gap between segments in mind.
+ *
+ * The result would look like this:
+ *
+ * 012d9000-0133e000 [heap]
+ * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f442e010000-7f443a800000 # reserved empty space
+ * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f444196c000-7f470a800000 # reserved empty space
+ * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
+ * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
+ * [...]
+ *
+ * The reserved space pointer is calculated to slice up the total reserved
+ * space into fixed fractions of address space for each segment, as specified
+ * in the SHMEM_RESIZE_RATIO array.
+ */
+static double SHMEM_RESIZE_RATIO[1] = {
+ 1.0, /* MAIN_SHMEM_SLOT */
+};
+
+/*
+ * Offset from the beginning of the reserved space, which indicates currently
+ * available range. New shared memory segments have to be allocated at this
+ * offset related to the reserved space.
+ */
+static Size reserved_offset = 0;
+
+/*
+ * Flag telling that we have decided to use huge pages.
+ *
+ * XXX: It's possible to use GetConfigOption("huge_pages_status", false, false)
+ * instead, but it feels like an overkill.
+ */
+static bool huge_pages_on = false;
+
static void *InternalIpcMemoryCreate(IpcMemoryKey memKey, Size size);
static void IpcMemoryDetach(int status, Datum shmaddr);
static void IpcMemoryDelete(int status, Datum shmId);
@@ -626,39 +686,198 @@ check_huge_page_size(int *newval, void **extra, GucSource source)
*
* This function will modify mapping size to the actual size of the allocation,
* if it ends up allocating a segment that is larger than requested.
+ *
+ * Note that we do not switch from huge pages to regular pages in this
+ * function, this decision was already made in ReserveAnonymousMemory and we
+ * stick to it.
*/
static void
-CreateAnonymousSegment(AnonymousMapping *mapping)
+CreateAnonymousSegment(AnonymousMapping *mapping, Pointer base)
{
Size allocsize = mapping->shmem_size;
void *ptr = MAP_FAILED;
int mmap_errno = 0;
+ int mmap_flags = PG_MMAP_FLAGS;
#ifndef MAP_HUGETLB
- /* PGSharedMemoryCreate should have dealt with this case */
- Assert(huge_pages != HUGE_PAGES_ON);
+ /* ReserveAnonymousMemory should have dealt with this case */
+ Assert(huge_pages != HUGE_PAGES_ON && !huge_pages_on);
#else
- if (huge_pages == HUGE_PAGES_ON || huge_pages == HUGE_PAGES_TRY)
+ if (huge_pages_on)
{
- /*
- * Round up the request size to a suitable large value.
- */
Size hugepagesize;
- int mmap_flags;
+ /* Make sure nothing is messed up */
+ Assert(huge_pages == HUGE_PAGES_ON || huge_pages == HUGE_PAGES_TRY);
+
+ /* Round up the request size to a suitable large value */
GetHugePageSize(&hugepagesize, &mmap_flags);
if (allocsize % hugepagesize != 0)
allocsize += hugepagesize - (allocsize % hugepagesize);
+ mmap_flags = PG_MMAP_FLAGS | mmap_flags;
+ }
+#endif
+
+ elog(DEBUG1, "segment[%s]: mmap(%zu) at address %p",
+ MappingName(mapping->shmem_segment), allocsize, base + reserved_offset);
+
+ /*
+ * Try to create mapping at an address out of the reserved range, which
+ * will allow to extend it later. Use reserved_offset to allocate the
+ * segment, then update currently available reserved range.
+ *
+ * If the last step has failed, fallback to the regular mapping
+ * creation and signal that shared buffers could not be resized without
+ * a restart.
+ */
+ ptr = mmap(base + reserved_offset, allocsize, PROT_READ | PROT_WRITE,
+ mmap_flags | MAP_FIXED, -1, 0);
+ mmap_errno = errno;
+
+ if (ptr == MAP_FAILED)
+ {
+ DebugMappings();
+ elog(DEBUG1, "segment[%s]: mmap(%zu) at address %p failed: %m, "
+ "fallback to the non-resizable allocation",
+ MappingName(mapping->shmem_segment), allocsize, base + reserved_offset);
+
ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
- PG_MMAP_FLAGS | mmap_flags, -1, 0);
+ PG_MMAP_FLAGS, -1, 0);
+ mmap_errno = errno;
+ }
+ else
+ {
+ Size total_reserved = (Size) MaxAvailableMemory * BLCKSZ;
+
+ reserved_offset += total_reserved * SHMEM_RESIZE_RATIO[next_free_segment];
+ }
+
+ if (ptr == MAP_FAILED)
+ {
+ errno = mmap_errno;
+ DebugMappings();
+ ereport(FATAL,
+ (errmsg("segment[%s]: could not map anonymous shared memory: %m",
+ MappingName(mapping->shmem_segment)),
+ (mmap_errno == ENOMEM) ?
+ errhint("This error usually means that PostgreSQL's request "
+ "for a shared memory segment exceeded available memory, "
+ "swap space, or huge pages. To reduce the request size "
+ "(currently %zu bytes), reduce PostgreSQL's shared "
+ "memory usage, perhaps by reducing \"shared_buffers\" or "
+ "\"max_connections\".",
+ allocsize) : 0));
+ }
+
+ mapping->shmem = ptr;
+ mapping->shmem_size = allocsize;
+}
+
+/*
+ * ReserveAnonymousMemory
+ *
+ * Reserve shared memory address space, from which shared memory segments are
+ * going to be sliced out. The goal of this exercise is to support segments
+ * resizing, for which we need a reserved space free of potential clashes with
+ * other mmap'd areas that are not under our control. Reservation is done via
+ * mmap, and will not allocate any memory until it will be actually used, and
+ * MAP_NORESERVE allows to make it not counting againt kernel reservation
+ * limits (e.g. in cgroups or for huge pages). Do not get confused because of
+ * MAP_NORESERVE -- we need to reserve some space, but not the actual memory,
+ * and that is that this flag is about.
+ *
+ * Note, that with MAP_NORESERVE a reservation with hugetlb will succeed even
+ * if there is actually not enough huge pages. Hence this function is
+ * responsible for deciding whether to use huge pages or not. To achieve that
+ * we need to probe first and try to allocate needed memory for all segments --
+ * if this succeeds, we unmap the probe segment and use hugetlb; if it fails,
+ * we proceed with the regular memory.
+ */
+void *
+ReserveAnonymousMemory(Size reserve_size)
+{
+ Size allocsize = reserve_size;
+ void *ptr = MAP_FAILED;
+ int mmap_errno = 0;
+
+ /* Complain if hugepages demanded but we can't possibly support them */
+#if !defined(MAP_HUGETLB)
+ if (huge_pages == HUGE_PAGES_ON)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("huge pages not supported on this platform")));
+#else
+ if (huge_pages == HUGE_PAGES_ON || huge_pages == HUGE_PAGES_TRY)
+ {
+ Size hugepagesize, total_size = 0;
+ int mmap_flags;
+
+ GetHugePageSize(&hugepagesize, &mmap_flags);
+
+ /*
+ * Figure out how much memory is needed for all segments, keeping in
+ * mind that for every segment this value will be rounding up by the
+ * huge page size. The resulting value will be used to probe memory and
+ * decide whether we will allocate huge pages or not.
+ *
+ * We could actually have a mix and match of segments with and without
+ * huge pages. But in that case we need to have multiple reservation
+ * spaces to use corresponding memory (hugetlb adress space reserved
+ * for hugetlb segments, regular memory for others), and it doesn't
+ * seem to worth the complexity for now.
+ */
+ for(int segment = 0; segment < ANON_MAPPINGS; segment++)
+ {
+ int numSemas;
+ Size segment_size = CalculateShmemSize(&numSemas, segment);
+
+ if (segment_size % hugepagesize != 0)
+ segment_size += hugepagesize - (segment_size % hugepagesize);
+
+ total_size += segment_size;
+ }
+
+ /* Map total amount of memory to test its availability. */
+ elog(DEBUG1, "reserving space: probe mmap(%zu) with MAP_HUGETLB",
+ total_size);
+ ptr = mmap(NULL, total_size, PROT_NONE,
+ PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
mmap_errno = errno;
if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
{
- DebugMappings();
- elog(DEBUG1, "segment[%s]: mmap(%zu) with MAP_HUGETLB failed, huge pages disabled: %m",
- MappingName(mapping->shmem_segment), allocsize);
+ /* No huge pages, we will go with the regular page size */
+ elog(DEBUG1, "reserving space: probe mmap(%zu) with MAP_HUGETLB "
+ "failed, huge pages disabled: %m", total_size);
+ }
+ else
+ {
+ /*
+ * All fine, unmap the temporary segment and proceed with reserving
+ * using huge pages.
+ */
+ if (munmap(ptr, total_size) < 0)
+ elog(LOG, "reservice space: munmap(%p, %zu) failed: %m",
+ ptr, total_size);
+
+ /* Round up the requested size to a suitable large value. */
+ if (allocsize % hugepagesize != 0)
+ allocsize += hugepagesize - (allocsize % hugepagesize);
+
+ elog(DEBUG1, "reserving space: mmap(%zu) with MAP_HUGETLB",
+ allocsize);
+ ptr = mmap(NULL, allocsize, PROT_NONE,
+ PG_MMAP_FLAGS | MAP_ANONYMOUS | MAP_NORESERVE | mmap_flags,
+ -1, 0);
+ mmap_errno = errno;
+
+ /* This should not happen, but handle errors anyway */
+ if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
+ {
+ elog(DEBUG1, "reserving space: mmap(%zu) with MAP_HUGETLB "
+ "failed, huge pages disabled: %m", allocsize);
+ }
}
}
#endif
@@ -666,10 +885,12 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
/*
* Report whether huge pages are in use. This needs to be tracked before
* the second mmap() call if attempting to use huge pages failed
- * previously.
+ * previously. At this point ptr is either pointing to the probe segment,
+ * if we couldn't mmap it, or the reservation space.
*/
SetConfigOption("huge_pages_status", (ptr == MAP_FAILED) ? "off" : "on",
PGC_INTERNAL, PGC_S_DYNAMIC_DEFAULT);
+ huge_pages_on = ptr != MAP_FAILED;
if (ptr == MAP_FAILED && huge_pages != HUGE_PAGES_ON)
{
@@ -677,10 +898,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
* Use the original size, not the rounded-up value, when falling back
* to non-huge pages.
*/
- allocsize = mapping->shmem_size;
- ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
- PG_MMAP_FLAGS, -1, 0);
- mmap_errno = errno;
+ allocsize = reserve_size;
+
+ elog(DEBUG1, "reserving space: mmap(%zu)", allocsize);
+ ptr = mmap(NULL, allocsize, PROT_NONE,
+ MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
}
if (ptr == MAP_FAILED)
@@ -688,20 +910,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
errno = mmap_errno;
DebugMappings();
ereport(FATAL,
- (errmsg("segment[%s]: could not map anonymous shared memory: %m",
- MappingName(mapping->shmem_segment)),
+ (errmsg("reserving space: could not map anonymous shared "
+ "memory: %m"),
(mmap_errno == ENOMEM) ?
errhint("This error usually means that PostgreSQL's request "
- "for a shared memory segment exceeded available memory, "
- "swap space, or huge pages. To reduce the request size "
- "(currently %zu bytes), reduce PostgreSQL's shared "
- "memory usage, perhaps by reducing \"shared_buffers\" or "
- "\"max_connections\".",
+ "for a reserved shared memory address space exceeded "
+ "available memory, swap space, or huge pages. To "
+ "reduce the request reservation size (currently %zu "
+ "bytes), reduce PostgreSQL's \"maximum_shared_buffers\".",
allocsize) : 0));
}
- mapping->shmem = ptr;
- mapping->shmem_size = allocsize;
+ return ptr;
}
/*
@@ -740,7 +960,7 @@ AnonymousShmemDetach(int status, Datum arg)
*/
PGShmemHeader *
PGSharedMemoryCreate(Size size,
- PGShmemHeader **shim)
+ PGShmemHeader **shim, Pointer base)
{
IpcMemoryKey NextShmemSegID;
void *memAddress;
@@ -760,14 +980,6 @@ PGSharedMemoryCreate(Size size,
errmsg("could not stat data directory \"%s\": %m",
DataDir)));
- /* Complain if hugepages demanded but we can't possibly support them */
-#if !defined(MAP_HUGETLB)
- if (huge_pages == HUGE_PAGES_ON)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("huge pages not supported on this platform")));
-#endif
-
/* For now, we don't support huge pages in SysV memory */
if (huge_pages == HUGE_PAGES_ON && shared_memory_type != SHMEM_TYPE_MMAP)
ereport(ERROR,
@@ -782,7 +994,7 @@ PGSharedMemoryCreate(Size size,
if (shared_memory_type == SHMEM_TYPE_MMAP)
{
/* On success, mapping data will be modified. */
- CreateAnonymousSegment(mapping);
+ CreateAnonymousSegment(mapping, base);
next_free_segment++;
diff --git a/src/backend/port/win32_shmem.c b/src/backend/port/win32_shmem.c
index 4dee856d6bd..ce719f1b412 100644
--- a/src/backend/port/win32_shmem.c
+++ b/src/backend/port/win32_shmem.c
@@ -205,7 +205,7 @@ EnableLockPagesPrivilege(int elevel)
*/
PGShmemHeader *
PGSharedMemoryCreate(Size size,
- PGShmemHeader **shim)
+ PGShmemHeader **shim, Pointer base)
{
void *memAddress;
PGShmemHeader *hdr;
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 8b38e985327..076888c0172 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -203,9 +203,12 @@ CreateSharedMemoryAndSemaphores(void)
PGShmemHeader *seghdr;
Size size;
int numSemas;
+ void *base;
Assert(!IsUnderPostmaster);
+ base = ReserveAnonymousMemory((Size) MaxAvailableMemory * BLCKSZ);
+
for(int segment = 0; segment < ANON_MAPPINGS; segment++)
{
/* Compute the size of the shared-memory block */
@@ -217,7 +220,7 @@ CreateSharedMemoryAndSemaphores(void)
*
* XXX: Do multiple shims are needed, one per segment?
*/
- seghdr = PGSharedMemoryCreate(size, &shim);
+ seghdr = PGSharedMemoryCreate(size, &shim, base);
/*
* Make sure that huge pages are never reported as "unknown" while the
diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c
index 2152aad97d9..1d42a5856c0 100644
--- a/src/backend/utils/init/globals.c
+++ b/src/backend/utils/init/globals.c
@@ -140,6 +140,7 @@ int max_parallel_maintenance_workers = 2;
* register background workers.
*/
int NBuffers = 16384;
+int MaxAvailableMemory = 131072;
int MaxConnections = 100;
int max_worker_processes = 8;
int max_parallel_workers = 8;
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 4eaeca89f2c..dede37f7905 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -2364,6 +2364,20 @@ struct config_int ConfigureNamesInt[] =
NULL, NULL, NULL
},
+ {
+ {"max_available_memory", PGC_SIGHUP, RESOURCES_MEM,
+ gettext_noop("Sets the upper limit for the shared_buffers value."),
+ gettext_noop("Shared memory could be resized at runtime, this "
+ "parameters sets the upper limit for it, beyond which "
+ "resizing would not be supported. Normally this value "
+ "would be the same as the total available memory."),
+ GUC_UNIT_BLOCKS
+ },
+ &MaxAvailableMemory,
+ 131072, 16, INT_MAX / 2,
+ NULL, NULL, NULL
+ },
+
{
{"vacuum_buffer_usage_limit", PGC_USERSET, RESOURCES_MEM,
gettext_noop("Sets the buffer pool size for VACUUM, ANALYZE, and autovacuum."),
diff --git a/src/include/storage/pg_shmem.h b/src/include/storage/pg_shmem.h
index 138078c29c5..4a83e255652 100644
--- a/src/include/storage/pg_shmem.h
+++ b/src/include/storage/pg_shmem.h
@@ -60,6 +60,7 @@ extern PGDLLIMPORT ShmemSegment Segments[ANON_MAPPINGS];
extern PGDLLIMPORT int shared_memory_type;
extern PGDLLIMPORT int huge_pages;
extern PGDLLIMPORT int huge_page_size;
+extern PGDLLIMPORT int MaxAvailableMemory;
/* Possible values for huge_pages and huge_pages_status */
typedef enum
@@ -100,10 +101,11 @@ extern void PGSharedMemoryNoReAttach(void);
#endif
extern PGShmemHeader *PGSharedMemoryCreate(Size size,
- PGShmemHeader **shim);
+ PGShmemHeader **shim, Pointer base);
extern bool PGSharedMemoryIsInUse(unsigned long id1, unsigned long id2);
extern void PGSharedMemoryDetach(void);
extern void GetHugePageSize(Size *hugepagesize, int *mmap_flags);
+void *ReserveAnonymousMemory(Size reserve_size);
/* The main segment, contains everything except buffer blocks and related data. */
#define MAIN_SHMEM_SEGMENT 0
--
2.45.1
--vninua6xybvzgrci
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename="v4-0003-Introduce-multiple-shmem-segments-for-shared-buff.patch"
^ permalink raw reply [nested|flat] 19+ messages in thread
* [PATCH v13 2/5] Remove specialized word-length popcount implementations.
@ 2026-01-23 23:31 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 19+ messages in thread
From: Nathan Bossart @ 2026-01-23 23:31 UTC (permalink / raw)
The uses of these functions do not justify the level of
micro-optimization we've done and may even hurt performance in some
cases (e.g., due to using function pointers). This commit removes
all architecture-specific implementations of pg_popcount{32,64}()
and converts the portable ones to inlined functions in
pg_bitutils.h. These inlined versions should produce the same code
as before (but inlined), so in theory this is a net gain for many
machines.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: Greg Burd <greg@burd.me>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
src/include/port/pg_bitutils.h | 75 +++++++++++++++++++++++-----------
src/port/pg_bitutils.c | 65 +----------------------------
src/port/pg_popcount_aarch64.c | 20 +++------
src/port/pg_popcount_x86.c | 43 +------------------
4 files changed, 59 insertions(+), 144 deletions(-)
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 20c11b79c61..789663edd93 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -276,46 +276,73 @@ pg_ceil_log2_64(uint64 num)
return pg_leftmost_one_pos64(num - 1) + 1;
}
-extern int pg_popcount32_portable(uint32 word);
-extern int pg_popcount64_portable(uint64 word);
extern uint64 pg_popcount_portable(const char *buf, int bytes);
extern uint64 pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask);
-#ifdef HAVE_X86_64_POPCNTQ
+#if defined(HAVE_X86_64_POPCNTQ) || defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
/*
- * Attempt to use SSE4.2 or AVX-512 instructions, but perform a runtime check
+ * Attempt to use specialized CPU instructions, but perform a runtime check
* first.
*/
-extern PGDLLIMPORT int (*pg_popcount32) (uint32 word);
-extern PGDLLIMPORT int (*pg_popcount64) (uint64 word);
extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
-#elif defined(USE_NEON)
-/* Use the Neon version of pg_popcount{32,64} without function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-
-/*
- * We can try to use an SVE-optimized pg_popcount() on some systems For that,
- * we do use a function pointer.
- */
-#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
-extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
-extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
#else
+/* Use a portable implementation -- no need for a function pointer. */
extern uint64 pg_popcount_optimized(const char *buf, int bytes);
extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+
#endif
-#else
-/* Use a portable implementation -- no need for a function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-extern uint64 pg_popcount_optimized(const char *buf, int bytes);
-extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+/*
+ * pg_popcount32
+ * Return the number of 1 bits set in word
+ */
+static inline int
+pg_popcount32(uint32 word)
+{
+#ifdef HAVE__BUILTIN_POPCOUNT
+ return __builtin_popcount(word);
+#else /* !HAVE__BUILTIN_POPCOUNT */
+ int result = 0;
+
+ while (word != 0)
+ {
+ result += pg_number_of_ones[word & 255];
+ word >>= 8;
+ }
+ return result;
+#endif /* HAVE__BUILTIN_POPCOUNT */
+}
+
+/*
+ * pg_popcount64
+ * Return the number of 1 bits set in word
+ */
+static inline int
+pg_popcount64(uint64 word)
+{
+#ifdef HAVE__BUILTIN_POPCOUNT
+#if SIZEOF_LONG == 8
+ return __builtin_popcountl(word);
+#elif SIZEOF_LONG_LONG == 8
+ return __builtin_popcountll(word);
+#else
+#error "cannot find integer of the same size as uint64_t"
#endif
+#else /* !HAVE__BUILTIN_POPCOUNT */
+ int result = 0;
+
+ while (word != 0)
+ {
+ result += pg_number_of_ones[word & 255];
+ word >>= 8;
+ }
+
+ return result;
+#endif /* HAVE__BUILTIN_POPCOUNT */
+}
/*
* Returns the number of 1-bits in buf.
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c
index bec06c06fc3..49b130f1306 100644
--- a/src/port/pg_bitutils.c
+++ b/src/port/pg_bitutils.c
@@ -96,56 +96,6 @@ const uint8 pg_number_of_ones[256] = {
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
};
-/*
- * pg_popcount32_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount32_portable(uint32 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
-/*
- * pg_popcount64_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount64_portable(uint64 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
/*
* pg_popcount_portable
* Returns the number of 1-bits in buf
@@ -163,7 +113,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++);
+ popcnt += pg_popcount64(*words++);
bytes -= 8;
}
@@ -197,7 +147,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++ & maskv);
+ popcnt += pg_popcount64(*words++ & maskv);
bytes -= 8;
}
@@ -220,17 +170,6 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
* actual external functions. The compiler should be able to inline the
* portable versions here.
*/
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount32_portable(word);
-}
-
-int
-pg_popcount64(uint64 word)
-{
- return pg_popcount64_portable(word);
-}
/*
* pg_popcount_optimized
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index ba57f2cd4bd..f474ef45510 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -292,21 +292,11 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
#endif /* ! USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
/*
- * pg_popcount32
+ * pg_popcount64_neon
* Return number of 1 bits in word
*/
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount64((uint64) word);
-}
-
-/*
- * pg_popcount64
- * Return number of 1 bits in word
- */
-int
-pg_popcount64(uint64 word)
+static inline int
+pg_popcount64_neon(uint64 word)
{
/*
* For some compilers, __builtin_popcountl() already emits Neon
@@ -383,7 +373,7 @@ pg_popcount_neon(const char *buf, int bytes)
*/
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
{
- popcnt += pg_popcount64(*((const uint64 *) buf));
+ popcnt += pg_popcount64_neon(*((const uint64 *) buf));
buf += sizeof(uint64);
}
@@ -465,7 +455,7 @@ pg_popcount_masked_neon(const char *buf, int bytes, bits8 mask)
*/
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
{
- popcnt += pg_popcount64(*((const uint64 *) buf) & mask64);
+ popcnt += pg_popcount64_neon(*((const uint64 *) buf) & mask64);
buf += sizeof(uint64);
}
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.c
index 7aebf69898b..6bce089432f 100644
--- a/src/port/pg_popcount_x86.c
+++ b/src/port/pg_popcount_x86.c
@@ -36,8 +36,6 @@
* operation, but in practice this is close enough, and "sse42" seems easier to
* follow than "popcnt" for these names.
*/
-static inline int pg_popcount32_sse42(uint32 word);
-static inline int pg_popcount64_sse42(uint64 word);
static uint64 pg_popcount_sse42(const char *buf, int bytes);
static uint64 pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask);
@@ -55,12 +53,8 @@ static uint64 pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask);
* what the current CPU supports) and then will call the pointer to fulfill the
* caller's request.
*/
-static int pg_popcount32_choose(uint32 word);
-static int pg_popcount64_choose(uint64 word);
static uint64 pg_popcount_choose(const char *buf, int bytes);
static uint64 pg_popcount_masked_choose(const char *buf, int bytes, bits8 mask);
-int (*pg_popcount32) (uint32 word) = pg_popcount32_choose;
-int (*pg_popcount64) (uint64 word) = pg_popcount64_choose;
uint64 (*pg_popcount_optimized) (const char *buf, int bytes) = pg_popcount_choose;
uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask) = pg_popcount_masked_choose;
@@ -157,7 +151,7 @@ pg_popcount_avx512_available(void)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
/*
- * These functions get called on the first call to pg_popcount32 etc.
+ * These functions get called on the first call to pg_popcount(), etc.
* They detect whether we can use the asm implementations, and replace
* the function pointers so that subsequent calls are routed directly to
* the chosen implementation.
@@ -167,15 +161,11 @@ choose_popcount_functions(void)
{
if (pg_popcount_sse42_available())
{
- pg_popcount32 = pg_popcount32_sse42;
- pg_popcount64 = pg_popcount64_sse42;
pg_popcount_optimized = pg_popcount_sse42;
pg_popcount_masked_optimized = pg_popcount_masked_sse42;
}
else
{
- pg_popcount32 = pg_popcount32_portable;
- pg_popcount64 = pg_popcount64_portable;
pg_popcount_optimized = pg_popcount_portable;
pg_popcount_masked_optimized = pg_popcount_masked_portable;
}
@@ -189,20 +179,6 @@ choose_popcount_functions(void)
#endif
}
-static int
-pg_popcount32_choose(uint32 word)
-{
- choose_popcount_functions();
- return pg_popcount32(word);
-}
-
-static int
-pg_popcount64_choose(uint64 word)
-{
- choose_popcount_functions();
- return pg_popcount64(word);
-}
-
static uint64
pg_popcount_choose(const char *buf, int bytes)
{
@@ -338,23 +314,6 @@ pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32_sse42
- * Return the number of 1 bits set in word
- */
-static inline int
-pg_popcount32_sse42(uint32 word)
-{
-#ifdef _MSC_VER
- return __popcnt(word);
-#else
- uint32 res;
-
-__asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc");
- return (int) res;
-#endif
-}
-
/*
* pg_popcount64_sse42
* Return the number of 1 bits set in word
--
2.50.1 (Apple Git-155)
--5NNwvjHYUnRaoEDo
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v13-0003-Remove-uses-of-popcount-builtins.patch
^ permalink raw reply [nested|flat] 19+ messages in thread
* [PATCH v5 3/3] Remove specialized word-length popcount implementations.
@ 2026-01-23 23:31 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 19+ messages in thread
From: Nathan Bossart @ 2026-01-23 23:31 UTC (permalink / raw)
---
src/include/port/pg_bitutils.h | 75 +++++++++++++++++++++++-----------
src/port/pg_bitutils.c | 69 ++-----------------------------
src/port/pg_popcount_aarch64.c | 25 ------------
src/port/pg_popcount_x86.c | 43 +------------------
4 files changed, 56 insertions(+), 156 deletions(-)
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 35761f509ec..a3e0f346ef6 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -276,46 +276,73 @@ pg_ceil_log2_64(uint64 num)
return pg_leftmost_one_pos64(num - 1) + 1;
}
-extern int pg_popcount32_portable(uint32 word);
-extern int pg_popcount64_portable(uint64 word);
extern uint64 pg_popcount_portable(const char *buf, int bytes);
extern uint64 pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask);
-#ifdef HAVE_X86_64_POPCNTQ
+#if defined(HAVE_X86_64_POPCNTQ) || defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
/*
- * Attempt to use SSE4.2 or AVX-512 instructions, but perform a runtime check
+ * Attempt to use specialized CPU instructions, but perform a runtime check
* first.
*/
-extern PGDLLIMPORT int (*pg_popcount32) (uint32 word);
-extern PGDLLIMPORT int (*pg_popcount64) (uint64 word);
extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
-#elif defined(USE_NEON)
-/* Use the Neon version of pg_popcount{32,64} without function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-
-/*
- * We can try to use an SVE-optimized pg_popcount() on some systems For that,
- * we do use a function pointer.
- */
-#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
-extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
-extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
#else
+/* Use a portable implementation -- no need for a function pointer. */
extern uint64 pg_popcount_optimized(const char *buf, int bytes);
extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+
#endif
-#else
-/* Use a portable implementation -- no need for a function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-extern uint64 pg_popcount_optimized(const char *buf, int bytes);
-extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+/*
+ * pg_popcount32
+ * Return the number of 1 bits set in word
+ */
+static inline int
+pg_popcount32(uint32 word)
+{
+#ifdef HAVE__BUILTIN_POPCOUNT
+ return __builtin_popcount(word);
+#else /* !HAVE__BUILTIN_POPCOUNT */
+ int result = 0;
+
+ while (word != 0)
+ {
+ result += pg_number_of_ones[word & 255];
+ word >>= 8;
+ }
+ return result;
+#endif /* HAVE__BUILTIN_POPCOUNT */
+}
+
+/*
+ * pg_popcount64
+ * Return the number of 1 bits set in word
+ */
+static inline int
+pg_popcount64(uint64 word)
+{
+#ifdef HAVE__BUILTIN_POPCOUNT
+#if SIZEOF_LONG == 8
+ return __builtin_popcountl(word);
+#elif SIZEOF_LONG_LONG == 8
+ return __builtin_popcountll(word);
+#else
+#error "cannot find integer of the same size as uint64_t"
#endif
+#else /* !HAVE__BUILTIN_POPCOUNT */
+ int result = 0;
+
+ while (word != 0)
+ {
+ result += pg_number_of_ones[word & 255];
+ word >>= 8;
+ }
+
+ return result;
+#endif /* HAVE__BUILTIN_POPCOUNT */
+}
/*
* Returns the number of 1-bits in buf.
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c
index ffda75825e5..f942c1be55a 100644
--- a/src/port/pg_bitutils.c
+++ b/src/port/pg_bitutils.c
@@ -96,56 +96,6 @@ const uint8 pg_number_of_ones[256] = {
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
};
-/*
- * pg_popcount32_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount32_portable(uint32 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
-/*
- * pg_popcount64_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount64_portable(uint64 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
/*
* pg_popcount_portable
* Returns the number of 1-bits in buf
@@ -163,7 +113,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++);
+ popcnt += pg_popcount64(*words++);
bytes -= 8;
}
@@ -177,7 +127,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 4)
{
- popcnt += pg_popcount32_portable(*words++);
+ popcnt += pg_popcount32(*words++);
bytes -= 4;
}
@@ -211,7 +161,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++ & maskv);
+ popcnt += pg_popcount64(*words++ & maskv);
bytes -= 8;
}
@@ -227,7 +177,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 4)
{
- popcnt += pg_popcount32_portable(*words++ & maskv);
+ popcnt += pg_popcount32(*words++ & maskv);
bytes -= 4;
}
@@ -250,17 +200,6 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
* actual external functions. The compiler should be able to inline the
* portable versions here.
*/
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount32_portable(word);
-}
-
-int
-pg_popcount64(uint64 word)
-{
- return pg_popcount64_portable(word);
-}
/*
* pg_popcount_optimized
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index ba57f2cd4bd..74f71593721 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -291,31 +291,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
#endif /* ! USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32
- * Return number of 1 bits in word
- */
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount64((uint64) word);
-}
-
-/*
- * pg_popcount64
- * Return number of 1 bits in word
- */
-int
-pg_popcount64(uint64 word)
-{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
- return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
-}
-
/*
* pg_popcount_neon
* Returns number of 1 bits in buf
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.c
index 0e98f532552..9fd8e18ed16 100644
--- a/src/port/pg_popcount_x86.c
+++ b/src/port/pg_popcount_x86.c
@@ -36,8 +36,6 @@
* operation, but in practice this is close enough, and "sse42" seems easier to
* follow than "popcnt" for these names.
*/
-static inline int pg_popcount32_sse42(uint32 word);
-static inline int pg_popcount64_sse42(uint64 word);
static uint64 pg_popcount_sse42(const char *buf, int bytes);
static uint64 pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask);
@@ -55,12 +53,8 @@ static uint64 pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask);
* what the current CPU supports) and then will call the pointer to fulfill the
* caller's request.
*/
-static int pg_popcount32_choose(uint32 word);
-static int pg_popcount64_choose(uint64 word);
static uint64 pg_popcount_choose(const char *buf, int bytes);
static uint64 pg_popcount_masked_choose(const char *buf, int bytes, bits8 mask);
-int (*pg_popcount32) (uint32 word) = pg_popcount32_choose;
-int (*pg_popcount64) (uint64 word) = pg_popcount64_choose;
uint64 (*pg_popcount_optimized) (const char *buf, int bytes) = pg_popcount_choose;
uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask) = pg_popcount_masked_choose;
@@ -157,7 +151,7 @@ pg_popcount_avx512_available(void)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
/*
- * These functions get called on the first call to pg_popcount32 etc.
+ * These functions get called on the first call to pg_popcount(), etc.
* They detect whether we can use the asm implementations, and replace
* the function pointers so that subsequent calls are routed directly to
* the chosen implementation.
@@ -167,15 +161,11 @@ choose_popcount_functions(void)
{
if (pg_popcount_sse42_available())
{
- pg_popcount32 = pg_popcount32_sse42;
- pg_popcount64 = pg_popcount64_sse42;
pg_popcount_optimized = pg_popcount_sse42;
pg_popcount_masked_optimized = pg_popcount_masked_sse42;
}
else
{
- pg_popcount32 = pg_popcount32_portable;
- pg_popcount64 = pg_popcount64_portable;
pg_popcount_optimized = pg_popcount_portable;
pg_popcount_masked_optimized = pg_popcount_masked_portable;
}
@@ -189,20 +179,6 @@ choose_popcount_functions(void)
#endif
}
-static int
-pg_popcount32_choose(uint32 word)
-{
- choose_popcount_functions();
- return pg_popcount32(word);
-}
-
-static int
-pg_popcount64_choose(uint64 word)
-{
- choose_popcount_functions();
- return pg_popcount64(word);
-}
-
static uint64
pg_popcount_choose(const char *buf, int bytes)
{
@@ -338,23 +314,6 @@ pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32_sse42
- * Return the number of 1 bits set in word
- */
-static inline int
-pg_popcount32_sse42(uint32 word)
-{
-#ifdef _MSC_VER
- return __popcnt(word);
-#else
- uint32 res;
-
-__asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc");
- return (int) res;
-#endif
-}
-
/*
* pg_popcount64_sse42
* Return the number of 1 bits set in word
--
2.50.1 (Apple Git-155)
--a4C7SGgzwrPm67dM--
^ permalink raw reply [nested|flat] 19+ messages in thread
* [PATCH v6 3/3] Remove specialized word-length popcount implementations.
@ 2026-01-23 23:31 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 19+ messages in thread
From: Nathan Bossart @ 2026-01-23 23:31 UTC (permalink / raw)
---
src/include/port/pg_bitutils.h | 75 +++++++++++++++++++++++-----------
src/port/pg_bitutils.c | 65 +----------------------------
src/port/pg_popcount_aarch64.c | 25 ------------
src/port/pg_popcount_x86.c | 43 +------------------
4 files changed, 54 insertions(+), 154 deletions(-)
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 35761f509ec..a3e0f346ef6 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -276,46 +276,73 @@ pg_ceil_log2_64(uint64 num)
return pg_leftmost_one_pos64(num - 1) + 1;
}
-extern int pg_popcount32_portable(uint32 word);
-extern int pg_popcount64_portable(uint64 word);
extern uint64 pg_popcount_portable(const char *buf, int bytes);
extern uint64 pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask);
-#ifdef HAVE_X86_64_POPCNTQ
+#if defined(HAVE_X86_64_POPCNTQ) || defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
/*
- * Attempt to use SSE4.2 or AVX-512 instructions, but perform a runtime check
+ * Attempt to use specialized CPU instructions, but perform a runtime check
* first.
*/
-extern PGDLLIMPORT int (*pg_popcount32) (uint32 word);
-extern PGDLLIMPORT int (*pg_popcount64) (uint64 word);
extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
-#elif defined(USE_NEON)
-/* Use the Neon version of pg_popcount{32,64} without function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-
-/*
- * We can try to use an SVE-optimized pg_popcount() on some systems For that,
- * we do use a function pointer.
- */
-#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
-extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
-extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
#else
+/* Use a portable implementation -- no need for a function pointer. */
extern uint64 pg_popcount_optimized(const char *buf, int bytes);
extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+
#endif
-#else
-/* Use a portable implementation -- no need for a function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-extern uint64 pg_popcount_optimized(const char *buf, int bytes);
-extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+/*
+ * pg_popcount32
+ * Return the number of 1 bits set in word
+ */
+static inline int
+pg_popcount32(uint32 word)
+{
+#ifdef HAVE__BUILTIN_POPCOUNT
+ return __builtin_popcount(word);
+#else /* !HAVE__BUILTIN_POPCOUNT */
+ int result = 0;
+
+ while (word != 0)
+ {
+ result += pg_number_of_ones[word & 255];
+ word >>= 8;
+ }
+ return result;
+#endif /* HAVE__BUILTIN_POPCOUNT */
+}
+
+/*
+ * pg_popcount64
+ * Return the number of 1 bits set in word
+ */
+static inline int
+pg_popcount64(uint64 word)
+{
+#ifdef HAVE__BUILTIN_POPCOUNT
+#if SIZEOF_LONG == 8
+ return __builtin_popcountl(word);
+#elif SIZEOF_LONG_LONG == 8
+ return __builtin_popcountll(word);
+#else
+#error "cannot find integer of the same size as uint64_t"
#endif
+#else /* !HAVE__BUILTIN_POPCOUNT */
+ int result = 0;
+
+ while (word != 0)
+ {
+ result += pg_number_of_ones[word & 255];
+ word >>= 8;
+ }
+
+ return result;
+#endif /* HAVE__BUILTIN_POPCOUNT */
+}
/*
* Returns the number of 1-bits in buf.
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c
index bec06c06fc3..49b130f1306 100644
--- a/src/port/pg_bitutils.c
+++ b/src/port/pg_bitutils.c
@@ -96,56 +96,6 @@ const uint8 pg_number_of_ones[256] = {
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
};
-/*
- * pg_popcount32_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount32_portable(uint32 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
-/*
- * pg_popcount64_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount64_portable(uint64 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
/*
* pg_popcount_portable
* Returns the number of 1-bits in buf
@@ -163,7 +113,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++);
+ popcnt += pg_popcount64(*words++);
bytes -= 8;
}
@@ -197,7 +147,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++ & maskv);
+ popcnt += pg_popcount64(*words++ & maskv);
bytes -= 8;
}
@@ -220,17 +170,6 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
* actual external functions. The compiler should be able to inline the
* portable versions here.
*/
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount32_portable(word);
-}
-
-int
-pg_popcount64(uint64 word)
-{
- return pg_popcount64_portable(word);
-}
/*
* pg_popcount_optimized
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index ba57f2cd4bd..74f71593721 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -291,31 +291,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
#endif /* ! USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32
- * Return number of 1 bits in word
- */
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount64((uint64) word);
-}
-
-/*
- * pg_popcount64
- * Return number of 1 bits in word
- */
-int
-pg_popcount64(uint64 word)
-{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
- return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
-}
-
/*
* pg_popcount_neon
* Returns number of 1 bits in buf
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.c
index 7aebf69898b..6bce089432f 100644
--- a/src/port/pg_popcount_x86.c
+++ b/src/port/pg_popcount_x86.c
@@ -36,8 +36,6 @@
* operation, but in practice this is close enough, and "sse42" seems easier to
* follow than "popcnt" for these names.
*/
-static inline int pg_popcount32_sse42(uint32 word);
-static inline int pg_popcount64_sse42(uint64 word);
static uint64 pg_popcount_sse42(const char *buf, int bytes);
static uint64 pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask);
@@ -55,12 +53,8 @@ static uint64 pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask);
* what the current CPU supports) and then will call the pointer to fulfill the
* caller's request.
*/
-static int pg_popcount32_choose(uint32 word);
-static int pg_popcount64_choose(uint64 word);
static uint64 pg_popcount_choose(const char *buf, int bytes);
static uint64 pg_popcount_masked_choose(const char *buf, int bytes, bits8 mask);
-int (*pg_popcount32) (uint32 word) = pg_popcount32_choose;
-int (*pg_popcount64) (uint64 word) = pg_popcount64_choose;
uint64 (*pg_popcount_optimized) (const char *buf, int bytes) = pg_popcount_choose;
uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask) = pg_popcount_masked_choose;
@@ -157,7 +151,7 @@ pg_popcount_avx512_available(void)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
/*
- * These functions get called on the first call to pg_popcount32 etc.
+ * These functions get called on the first call to pg_popcount(), etc.
* They detect whether we can use the asm implementations, and replace
* the function pointers so that subsequent calls are routed directly to
* the chosen implementation.
@@ -167,15 +161,11 @@ choose_popcount_functions(void)
{
if (pg_popcount_sse42_available())
{
- pg_popcount32 = pg_popcount32_sse42;
- pg_popcount64 = pg_popcount64_sse42;
pg_popcount_optimized = pg_popcount_sse42;
pg_popcount_masked_optimized = pg_popcount_masked_sse42;
}
else
{
- pg_popcount32 = pg_popcount32_portable;
- pg_popcount64 = pg_popcount64_portable;
pg_popcount_optimized = pg_popcount_portable;
pg_popcount_masked_optimized = pg_popcount_masked_portable;
}
@@ -189,20 +179,6 @@ choose_popcount_functions(void)
#endif
}
-static int
-pg_popcount32_choose(uint32 word)
-{
- choose_popcount_functions();
- return pg_popcount32(word);
-}
-
-static int
-pg_popcount64_choose(uint64 word)
-{
- choose_popcount_functions();
- return pg_popcount64(word);
-}
-
static uint64
pg_popcount_choose(const char *buf, int bytes)
{
@@ -338,23 +314,6 @@ pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32_sse42
- * Return the number of 1 bits set in word
- */
-static inline int
-pg_popcount32_sse42(uint32 word)
-{
-#ifdef _MSC_VER
- return __popcnt(word);
-#else
- uint32 res;
-
-__asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc");
- return (int) res;
-#endif
-}
-
/*
* pg_popcount64_sse42
* Return the number of 1 bits set in word
--
2.50.1 (Apple Git-155)
--vgXyZptCXHDBThGI--
^ permalink raw reply [nested|flat] 19+ messages in thread
* [PATCH v12 2/4] Remove specialized word-length popcount implementations.
@ 2026-01-23 23:31 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 19+ messages in thread
From: Nathan Bossart @ 2026-01-23 23:31 UTC (permalink / raw)
The uses of these functions do not justify the level of
micro-optimization we've done and may even hurt performance in some
cases (e.g., due to using function pointers). This commit removes
all architecture-specific implementations of pg_popcount{32,64}()
and converts the portable ones to inlined functions in
pg_bitutils.h. These inlined versions should produce the same code
as before (but inlined), so in theory this is a net gain for many
machines.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: Greg Burd <greg@burd.me>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
src/include/port/pg_bitutils.h | 75 +++++++++++++++++++++++-----------
src/port/pg_bitutils.c | 65 +----------------------------
src/port/pg_popcount_aarch64.c | 20 +++------
src/port/pg_popcount_x86.c | 43 +------------------
4 files changed, 59 insertions(+), 144 deletions(-)
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 20c11b79c61..789663edd93 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -276,46 +276,73 @@ pg_ceil_log2_64(uint64 num)
return pg_leftmost_one_pos64(num - 1) + 1;
}
-extern int pg_popcount32_portable(uint32 word);
-extern int pg_popcount64_portable(uint64 word);
extern uint64 pg_popcount_portable(const char *buf, int bytes);
extern uint64 pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask);
-#ifdef HAVE_X86_64_POPCNTQ
+#if defined(HAVE_X86_64_POPCNTQ) || defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
/*
- * Attempt to use SSE4.2 or AVX-512 instructions, but perform a runtime check
+ * Attempt to use specialized CPU instructions, but perform a runtime check
* first.
*/
-extern PGDLLIMPORT int (*pg_popcount32) (uint32 word);
-extern PGDLLIMPORT int (*pg_popcount64) (uint64 word);
extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
-#elif defined(USE_NEON)
-/* Use the Neon version of pg_popcount{32,64} without function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-
-/*
- * We can try to use an SVE-optimized pg_popcount() on some systems For that,
- * we do use a function pointer.
- */
-#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
-extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
-extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
#else
+/* Use a portable implementation -- no need for a function pointer. */
extern uint64 pg_popcount_optimized(const char *buf, int bytes);
extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+
#endif
-#else
-/* Use a portable implementation -- no need for a function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-extern uint64 pg_popcount_optimized(const char *buf, int bytes);
-extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+/*
+ * pg_popcount32
+ * Return the number of 1 bits set in word
+ */
+static inline int
+pg_popcount32(uint32 word)
+{
+#ifdef HAVE__BUILTIN_POPCOUNT
+ return __builtin_popcount(word);
+#else /* !HAVE__BUILTIN_POPCOUNT */
+ int result = 0;
+
+ while (word != 0)
+ {
+ result += pg_number_of_ones[word & 255];
+ word >>= 8;
+ }
+ return result;
+#endif /* HAVE__BUILTIN_POPCOUNT */
+}
+
+/*
+ * pg_popcount64
+ * Return the number of 1 bits set in word
+ */
+static inline int
+pg_popcount64(uint64 word)
+{
+#ifdef HAVE__BUILTIN_POPCOUNT
+#if SIZEOF_LONG == 8
+ return __builtin_popcountl(word);
+#elif SIZEOF_LONG_LONG == 8
+ return __builtin_popcountll(word);
+#else
+#error "cannot find integer of the same size as uint64_t"
#endif
+#else /* !HAVE__BUILTIN_POPCOUNT */
+ int result = 0;
+
+ while (word != 0)
+ {
+ result += pg_number_of_ones[word & 255];
+ word >>= 8;
+ }
+
+ return result;
+#endif /* HAVE__BUILTIN_POPCOUNT */
+}
/*
* Returns the number of 1-bits in buf.
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c
index bec06c06fc3..49b130f1306 100644
--- a/src/port/pg_bitutils.c
+++ b/src/port/pg_bitutils.c
@@ -96,56 +96,6 @@ const uint8 pg_number_of_ones[256] = {
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
};
-/*
- * pg_popcount32_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount32_portable(uint32 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
-/*
- * pg_popcount64_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount64_portable(uint64 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
/*
* pg_popcount_portable
* Returns the number of 1-bits in buf
@@ -163,7 +113,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++);
+ popcnt += pg_popcount64(*words++);
bytes -= 8;
}
@@ -197,7 +147,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++ & maskv);
+ popcnt += pg_popcount64(*words++ & maskv);
bytes -= 8;
}
@@ -220,17 +170,6 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
* actual external functions. The compiler should be able to inline the
* portable versions here.
*/
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount32_portable(word);
-}
-
-int
-pg_popcount64(uint64 word)
-{
- return pg_popcount64_portable(word);
-}
/*
* pg_popcount_optimized
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index ba57f2cd4bd..f474ef45510 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -292,21 +292,11 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
#endif /* ! USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
/*
- * pg_popcount32
+ * pg_popcount64_neon
* Return number of 1 bits in word
*/
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount64((uint64) word);
-}
-
-/*
- * pg_popcount64
- * Return number of 1 bits in word
- */
-int
-pg_popcount64(uint64 word)
+static inline int
+pg_popcount64_neon(uint64 word)
{
/*
* For some compilers, __builtin_popcountl() already emits Neon
@@ -383,7 +373,7 @@ pg_popcount_neon(const char *buf, int bytes)
*/
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
{
- popcnt += pg_popcount64(*((const uint64 *) buf));
+ popcnt += pg_popcount64_neon(*((const uint64 *) buf));
buf += sizeof(uint64);
}
@@ -465,7 +455,7 @@ pg_popcount_masked_neon(const char *buf, int bytes, bits8 mask)
*/
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
{
- popcnt += pg_popcount64(*((const uint64 *) buf) & mask64);
+ popcnt += pg_popcount64_neon(*((const uint64 *) buf) & mask64);
buf += sizeof(uint64);
}
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.c
index 7aebf69898b..6bce089432f 100644
--- a/src/port/pg_popcount_x86.c
+++ b/src/port/pg_popcount_x86.c
@@ -36,8 +36,6 @@
* operation, but in practice this is close enough, and "sse42" seems easier to
* follow than "popcnt" for these names.
*/
-static inline int pg_popcount32_sse42(uint32 word);
-static inline int pg_popcount64_sse42(uint64 word);
static uint64 pg_popcount_sse42(const char *buf, int bytes);
static uint64 pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask);
@@ -55,12 +53,8 @@ static uint64 pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask);
* what the current CPU supports) and then will call the pointer to fulfill the
* caller's request.
*/
-static int pg_popcount32_choose(uint32 word);
-static int pg_popcount64_choose(uint64 word);
static uint64 pg_popcount_choose(const char *buf, int bytes);
static uint64 pg_popcount_masked_choose(const char *buf, int bytes, bits8 mask);
-int (*pg_popcount32) (uint32 word) = pg_popcount32_choose;
-int (*pg_popcount64) (uint64 word) = pg_popcount64_choose;
uint64 (*pg_popcount_optimized) (const char *buf, int bytes) = pg_popcount_choose;
uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask) = pg_popcount_masked_choose;
@@ -157,7 +151,7 @@ pg_popcount_avx512_available(void)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
/*
- * These functions get called on the first call to pg_popcount32 etc.
+ * These functions get called on the first call to pg_popcount(), etc.
* They detect whether we can use the asm implementations, and replace
* the function pointers so that subsequent calls are routed directly to
* the chosen implementation.
@@ -167,15 +161,11 @@ choose_popcount_functions(void)
{
if (pg_popcount_sse42_available())
{
- pg_popcount32 = pg_popcount32_sse42;
- pg_popcount64 = pg_popcount64_sse42;
pg_popcount_optimized = pg_popcount_sse42;
pg_popcount_masked_optimized = pg_popcount_masked_sse42;
}
else
{
- pg_popcount32 = pg_popcount32_portable;
- pg_popcount64 = pg_popcount64_portable;
pg_popcount_optimized = pg_popcount_portable;
pg_popcount_masked_optimized = pg_popcount_masked_portable;
}
@@ -189,20 +179,6 @@ choose_popcount_functions(void)
#endif
}
-static int
-pg_popcount32_choose(uint32 word)
-{
- choose_popcount_functions();
- return pg_popcount32(word);
-}
-
-static int
-pg_popcount64_choose(uint64 word)
-{
- choose_popcount_functions();
- return pg_popcount64(word);
-}
-
static uint64
pg_popcount_choose(const char *buf, int bytes)
{
@@ -338,23 +314,6 @@ pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32_sse42
- * Return the number of 1 bits set in word
- */
-static inline int
-pg_popcount32_sse42(uint32 word)
-{
-#ifdef _MSC_VER
- return __popcnt(word);
-#else
- uint32 res;
-
-__asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc");
- return (int) res;
-#endif
-}
-
/*
* pg_popcount64_sse42
* Return the number of 1 bits set in word
--
2.50.1 (Apple Git-155)
--AFLuD1w+kMRlv6Zk
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v12-0003-Remove-uses-of-popcount-builtins.patch
^ permalink raw reply [nested|flat] 19+ messages in thread
* [PATCH v8 2/3] Remove specialized word-length popcount implementations.
@ 2026-01-23 23:31 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 19+ messages in thread
From: Nathan Bossart @ 2026-01-23 23:31 UTC (permalink / raw)
The uses of these functions do not justify the level of
micro-optimization we've done and may even hurt performance in some
cases (e.g., due to using function pointers). This commit removes
all architecture-specific implementations of pg_popcount{32,64}()
and converts the portable ones to inlined functions in
pg_bitutils.h. These inlined versions should produce the same code
as before (but inlined), so in theory this is a net gain for many
machines. As an exception, for x86-64/gcc without sse4.2/popcnt,
we use a plain C version to ensure inlining because
__builtin_popcount() and __builtin_popcountl() generate function
calls for that configuration. Our tests indicate this is still a
net win.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: Greg Burd <greg@burd.me>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
src/include/port/pg_bitutils.h | 83 ++++++++++++++++++++++++----------
src/port/pg_bitutils.c | 65 +-------------------------
src/port/pg_popcount_aarch64.c | 25 ----------
src/port/pg_popcount_x86.c | 43 +-----------------
4 files changed, 63 insertions(+), 153 deletions(-)
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index c3049d71894..08b9abf5fe7 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -276,46 +276,83 @@ pg_ceil_log2_64(uint64 num)
return pg_leftmost_one_pos64(num - 1) + 1;
}
-extern int pg_popcount32_portable(uint32 word);
-extern int pg_popcount64_portable(uint64 word);
extern uint64 pg_popcount_portable(const char *buf, int bytes);
extern uint64 pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask);
-#ifdef HAVE_X86_64_POPCNTQ
+#if defined(HAVE_X86_64_POPCNTQ) || defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
/*
- * Attempt to use SSE4.2 or AVX-512 instructions, but perform a runtime check
+ * Attempt to use specialized CPU instructions, but perform a runtime check
* first.
*/
-extern PGDLLIMPORT int (*pg_popcount32) (uint32 word);
-extern PGDLLIMPORT int (*pg_popcount64) (uint64 word);
extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
-#elif defined(USE_NEON)
-/* Use the Neon version of pg_popcount{32,64} without function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-
-/*
- * We can try to use an SVE-optimized pg_popcount() on some systems For that,
- * we do use a function pointer.
- */
-#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
-extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
-extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
#else
+/* Use a portable implementation -- no need for a function pointer. */
extern uint64 pg_popcount_optimized(const char *buf, int bytes);
extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+
#endif
+/*
+ * pg_popcount32
+ * Return the number of 1 bits set in word
+ *
+ * Plain C version adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ */
+static inline int
+pg_popcount32(uint32 word)
+{
+ /*
+ * On x86, gcc generates a function call for this built-in unless the
+ * popcnt instruction is available, so we use the plain C version in that
+ * case to ensure inlining.
+ */
+#if defined(HAVE__BUILTIN_POPCOUNT) && (defined(__POPCNT__) || !defined(__x86_64__))
+ return __builtin_popcount(word);
+#elif defined(_MSC_VER)
+ return __popcnt(word);
#else
-/* Use a portable implementation -- no need for a function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-extern uint64 pg_popcount_optimized(const char *buf, int bytes);
-extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+ word -= (word >> 1) & 0x55555555;
+ word = (word & 0x33333333) + ((word >> 2) & 0x33333333);
+ return ((word + (word >> 4) & 0xf0f0f0f) * 0x1010101) >> 24;
+#endif
+}
+/*
+ * pg_popcount64
+ * Return the number of 1 bits set in word
+ *
+ * Plain C version adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ */
+static inline int
+pg_popcount64(uint64 word)
+{
+ /*
+ * On x86, gcc generates a function call for this built-in unless the
+ * popcnt instruction is available, so we use the plain C version in that
+ * case to ensure inlining.
+ */
+#if defined(HAVE__BUILTIN_POPCOUNT) && (defined(__POPCNT__) || !defined(__x86_64__))
+#if SIZEOF_LONG == 8
+ return __builtin_popcountl(word);
+#elif SIZEOF_LONG_LONG == 8
+ return __builtin_popcountll(word);
+#else
+#error "cannot find integer of the same size as uint64_t"
#endif
+#elif defined(_MSC_VER)
+ return __popcnt64(word);
+#else
+ word -= (word >> 1) & UINT64CONST(0x5555555555555555);
+ word = (word & UINT64CONST(0x3333333333333333)) +
+ ((word >> 2) & UINT64CONST(0x3333333333333333));
+ word = (word + (word >> 4)) & UINT64CONST(0xf0f0f0f0f0f0f0f);
+ return (word * UINT64CONST(0x101010101010101)) >> 56;
+#endif
+}
/*
* Returns the number of 1-bits in buf.
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c
index bec06c06fc3..49b130f1306 100644
--- a/src/port/pg_bitutils.c
+++ b/src/port/pg_bitutils.c
@@ -96,56 +96,6 @@ const uint8 pg_number_of_ones[256] = {
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
};
-/*
- * pg_popcount32_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount32_portable(uint32 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
-/*
- * pg_popcount64_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount64_portable(uint64 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
/*
* pg_popcount_portable
* Returns the number of 1-bits in buf
@@ -163,7 +113,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++);
+ popcnt += pg_popcount64(*words++);
bytes -= 8;
}
@@ -197,7 +147,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++ & maskv);
+ popcnt += pg_popcount64(*words++ & maskv);
bytes -= 8;
}
@@ -220,17 +170,6 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
* actual external functions. The compiler should be able to inline the
* portable versions here.
*/
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount32_portable(word);
-}
-
-int
-pg_popcount64(uint64 word)
-{
- return pg_popcount64_portable(word);
-}
/*
* pg_popcount_optimized
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index ba57f2cd4bd..74f71593721 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -291,31 +291,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
#endif /* ! USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32
- * Return number of 1 bits in word
- */
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount64((uint64) word);
-}
-
-/*
- * pg_popcount64
- * Return number of 1 bits in word
- */
-int
-pg_popcount64(uint64 word)
-{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
- return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
-}
-
/*
* pg_popcount_neon
* Returns number of 1 bits in buf
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.c
index 7aebf69898b..6bce089432f 100644
--- a/src/port/pg_popcount_x86.c
+++ b/src/port/pg_popcount_x86.c
@@ -36,8 +36,6 @@
* operation, but in practice this is close enough, and "sse42" seems easier to
* follow than "popcnt" for these names.
*/
-static inline int pg_popcount32_sse42(uint32 word);
-static inline int pg_popcount64_sse42(uint64 word);
static uint64 pg_popcount_sse42(const char *buf, int bytes);
static uint64 pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask);
@@ -55,12 +53,8 @@ static uint64 pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask);
* what the current CPU supports) and then will call the pointer to fulfill the
* caller's request.
*/
-static int pg_popcount32_choose(uint32 word);
-static int pg_popcount64_choose(uint64 word);
static uint64 pg_popcount_choose(const char *buf, int bytes);
static uint64 pg_popcount_masked_choose(const char *buf, int bytes, bits8 mask);
-int (*pg_popcount32) (uint32 word) = pg_popcount32_choose;
-int (*pg_popcount64) (uint64 word) = pg_popcount64_choose;
uint64 (*pg_popcount_optimized) (const char *buf, int bytes) = pg_popcount_choose;
uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask) = pg_popcount_masked_choose;
@@ -157,7 +151,7 @@ pg_popcount_avx512_available(void)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
/*
- * These functions get called on the first call to pg_popcount32 etc.
+ * These functions get called on the first call to pg_popcount(), etc.
* They detect whether we can use the asm implementations, and replace
* the function pointers so that subsequent calls are routed directly to
* the chosen implementation.
@@ -167,15 +161,11 @@ choose_popcount_functions(void)
{
if (pg_popcount_sse42_available())
{
- pg_popcount32 = pg_popcount32_sse42;
- pg_popcount64 = pg_popcount64_sse42;
pg_popcount_optimized = pg_popcount_sse42;
pg_popcount_masked_optimized = pg_popcount_masked_sse42;
}
else
{
- pg_popcount32 = pg_popcount32_portable;
- pg_popcount64 = pg_popcount64_portable;
pg_popcount_optimized = pg_popcount_portable;
pg_popcount_masked_optimized = pg_popcount_masked_portable;
}
@@ -189,20 +179,6 @@ choose_popcount_functions(void)
#endif
}
-static int
-pg_popcount32_choose(uint32 word)
-{
- choose_popcount_functions();
- return pg_popcount32(word);
-}
-
-static int
-pg_popcount64_choose(uint64 word)
-{
- choose_popcount_functions();
- return pg_popcount64(word);
-}
-
static uint64
pg_popcount_choose(const char *buf, int bytes)
{
@@ -338,23 +314,6 @@ pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32_sse42
- * Return the number of 1 bits set in word
- */
-static inline int
-pg_popcount32_sse42(uint32 word)
-{
-#ifdef _MSC_VER
- return __popcnt(word);
-#else
- uint32 res;
-
-__asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc");
- return (int) res;
-#endif
-}
-
/*
* pg_popcount64_sse42
* Return the number of 1 bits set in word
--
2.50.1 (Apple Git-155)
--dIDo5IfS/hVe25hD
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v8-0003-Make-use-of-pg_popcount-in-more-places.patch
^ permalink raw reply [nested|flat] 19+ messages in thread
* [PATCH v9 2/3] Remove specialized word-length popcount implementations.
@ 2026-01-23 23:31 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 19+ messages in thread
From: Nathan Bossart @ 2026-01-23 23:31 UTC (permalink / raw)
The uses of these functions do not justify the level of
micro-optimization we've done and may even hurt performance in some
cases (e.g., due to using function pointers). This commit removes
all architecture-specific implementations of pg_popcount{32,64}()
and converts the portable ones to inlined functions in
pg_bitutils.h. These inlined versions should produce the same code
as before (but inlined), so in theory this is a net gain for many
machines. As an exception, for x86-64/gcc without sse4.2/popcnt,
we use a plain C version to ensure inlining because
__builtin_popcount() and __builtin_popcountl() generate function
calls for that configuration. Our tests indicate this is still a
net win.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: Greg Burd <greg@burd.me>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
src/include/port/pg_bitutils.h | 83 ++++++++++++++++++++++++----------
src/port/pg_bitutils.c | 65 +-------------------------
src/port/pg_popcount_aarch64.c | 25 ----------
src/port/pg_popcount_x86.c | 43 +-----------------
4 files changed, 63 insertions(+), 153 deletions(-)
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index c3049d71894..a5c2673db51 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -276,46 +276,83 @@ pg_ceil_log2_64(uint64 num)
return pg_leftmost_one_pos64(num - 1) + 1;
}
-extern int pg_popcount32_portable(uint32 word);
-extern int pg_popcount64_portable(uint64 word);
extern uint64 pg_popcount_portable(const char *buf, int bytes);
extern uint64 pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask);
-#ifdef HAVE_X86_64_POPCNTQ
+#if defined(HAVE_X86_64_POPCNTQ) || defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
/*
- * Attempt to use SSE4.2 or AVX-512 instructions, but perform a runtime check
+ * Attempt to use specialized CPU instructions, but perform a runtime check
* first.
*/
-extern PGDLLIMPORT int (*pg_popcount32) (uint32 word);
-extern PGDLLIMPORT int (*pg_popcount64) (uint64 word);
extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
-#elif defined(USE_NEON)
-/* Use the Neon version of pg_popcount{32,64} without function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-
-/*
- * We can try to use an SVE-optimized pg_popcount() on some systems For that,
- * we do use a function pointer.
- */
-#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
-extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
-extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
#else
+/* Use a portable implementation -- no need for a function pointer. */
extern uint64 pg_popcount_optimized(const char *buf, int bytes);
extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+
#endif
+/*
+ * pg_popcount32
+ * Return the number of 1 bits set in word
+ *
+ * Plain C version adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ */
+static inline int
+pg_popcount32(uint32 word)
+{
+ /*
+ * On x86, gcc generates a function call for this built-in unless the
+ * popcnt instruction is available, so we use the plain C version in that
+ * case to ensure inlining.
+ */
+#if defined(HAVE__BUILTIN_POPCOUNT) && (defined(__POPCNT__) || !defined(__x86_64__))
+ return __builtin_popcount(word);
+#elif defined(_MSC_VER)
+ return __popcnt(word);
#else
-/* Use a portable implementation -- no need for a function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-extern uint64 pg_popcount_optimized(const char *buf, int bytes);
-extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+ word -= (word >> 1) & 0x55555555;
+ word = (word & 0x33333333) + ((word >> 2) & 0x33333333);
+ return (((word + (word >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
+#endif
+}
+/*
+ * pg_popcount64
+ * Return the number of 1 bits set in word
+ *
+ * Plain C version adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ */
+static inline int
+pg_popcount64(uint64 word)
+{
+ /*
+ * On x86, gcc generates a function call for this built-in unless the
+ * popcnt instruction is available, so we use the plain C version in that
+ * case to ensure inlining.
+ */
+#if defined(HAVE__BUILTIN_POPCOUNT) && (defined(__POPCNT__) || !defined(__x86_64__))
+#if SIZEOF_LONG == 8
+ return __builtin_popcountl(word);
+#elif SIZEOF_LONG_LONG == 8
+ return __builtin_popcountll(word);
+#else
+#error "cannot find integer of the same size as uint64_t"
#endif
+#elif defined(_MSC_VER)
+ return __popcnt64(word);
+#else
+ word -= (word >> 1) & UINT64CONST(0x5555555555555555);
+ word = (word & UINT64CONST(0x3333333333333333)) +
+ ((word >> 2) & UINT64CONST(0x3333333333333333));
+ word = (word + (word >> 4)) & UINT64CONST(0xf0f0f0f0f0f0f0f);
+ return (word * UINT64CONST(0x101010101010101)) >> 56;
+#endif
+}
/*
* Returns the number of 1-bits in buf.
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c
index bec06c06fc3..49b130f1306 100644
--- a/src/port/pg_bitutils.c
+++ b/src/port/pg_bitutils.c
@@ -96,56 +96,6 @@ const uint8 pg_number_of_ones[256] = {
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
};
-/*
- * pg_popcount32_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount32_portable(uint32 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
-/*
- * pg_popcount64_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount64_portable(uint64 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
/*
* pg_popcount_portable
* Returns the number of 1-bits in buf
@@ -163,7 +113,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++);
+ popcnt += pg_popcount64(*words++);
bytes -= 8;
}
@@ -197,7 +147,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++ & maskv);
+ popcnt += pg_popcount64(*words++ & maskv);
bytes -= 8;
}
@@ -220,17 +170,6 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
* actual external functions. The compiler should be able to inline the
* portable versions here.
*/
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount32_portable(word);
-}
-
-int
-pg_popcount64(uint64 word)
-{
- return pg_popcount64_portable(word);
-}
/*
* pg_popcount_optimized
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index ba57f2cd4bd..74f71593721 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -291,31 +291,6 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
#endif /* ! USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32
- * Return number of 1 bits in word
- */
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount64((uint64) word);
-}
-
-/*
- * pg_popcount64
- * Return number of 1 bits in word
- */
-int
-pg_popcount64(uint64 word)
-{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
- return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
-}
-
/*
* pg_popcount_neon
* Returns number of 1 bits in buf
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.c
index 7aebf69898b..6bce089432f 100644
--- a/src/port/pg_popcount_x86.c
+++ b/src/port/pg_popcount_x86.c
@@ -36,8 +36,6 @@
* operation, but in practice this is close enough, and "sse42" seems easier to
* follow than "popcnt" for these names.
*/
-static inline int pg_popcount32_sse42(uint32 word);
-static inline int pg_popcount64_sse42(uint64 word);
static uint64 pg_popcount_sse42(const char *buf, int bytes);
static uint64 pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask);
@@ -55,12 +53,8 @@ static uint64 pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask);
* what the current CPU supports) and then will call the pointer to fulfill the
* caller's request.
*/
-static int pg_popcount32_choose(uint32 word);
-static int pg_popcount64_choose(uint64 word);
static uint64 pg_popcount_choose(const char *buf, int bytes);
static uint64 pg_popcount_masked_choose(const char *buf, int bytes, bits8 mask);
-int (*pg_popcount32) (uint32 word) = pg_popcount32_choose;
-int (*pg_popcount64) (uint64 word) = pg_popcount64_choose;
uint64 (*pg_popcount_optimized) (const char *buf, int bytes) = pg_popcount_choose;
uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask) = pg_popcount_masked_choose;
@@ -157,7 +151,7 @@ pg_popcount_avx512_available(void)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
/*
- * These functions get called on the first call to pg_popcount32 etc.
+ * These functions get called on the first call to pg_popcount(), etc.
* They detect whether we can use the asm implementations, and replace
* the function pointers so that subsequent calls are routed directly to
* the chosen implementation.
@@ -167,15 +161,11 @@ choose_popcount_functions(void)
{
if (pg_popcount_sse42_available())
{
- pg_popcount32 = pg_popcount32_sse42;
- pg_popcount64 = pg_popcount64_sse42;
pg_popcount_optimized = pg_popcount_sse42;
pg_popcount_masked_optimized = pg_popcount_masked_sse42;
}
else
{
- pg_popcount32 = pg_popcount32_portable;
- pg_popcount64 = pg_popcount64_portable;
pg_popcount_optimized = pg_popcount_portable;
pg_popcount_masked_optimized = pg_popcount_masked_portable;
}
@@ -189,20 +179,6 @@ choose_popcount_functions(void)
#endif
}
-static int
-pg_popcount32_choose(uint32 word)
-{
- choose_popcount_functions();
- return pg_popcount32(word);
-}
-
-static int
-pg_popcount64_choose(uint64 word)
-{
- choose_popcount_functions();
- return pg_popcount64(word);
-}
-
static uint64
pg_popcount_choose(const char *buf, int bytes)
{
@@ -338,23 +314,6 @@ pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32_sse42
- * Return the number of 1 bits set in word
- */
-static inline int
-pg_popcount32_sse42(uint32 word)
-{
-#ifdef _MSC_VER
- return __popcnt(word);
-#else
- uint32 res;
-
-__asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc");
- return (int) res;
-#endif
-}
-
/*
* pg_popcount64_sse42
* Return the number of 1 bits set in word
--
2.50.1 (Apple Git-155)
--oEt0RkbzovU82iL7
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v9-0003-Make-use-of-pg_popcount-in-more-places.patch
^ permalink raw reply [nested|flat] 19+ messages in thread
* [PATCH v10 2/3] Remove specialized word-length popcount implementations.
@ 2026-01-23 23:31 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 19+ messages in thread
From: Nathan Bossart @ 2026-01-23 23:31 UTC (permalink / raw)
The uses of these functions do not justify the level of
micro-optimization we've done and may even hurt performance in some
cases (e.g., due to using function pointers). This commit removes
all architecture-specific implementations of pg_popcount{32,64}()
and converts the portable ones to inlined functions in
pg_bitutils.h. These inlined versions should produce the same code
as before (but inlined), so in theory this is a net gain for many
machines. As an exception, for x86-64/gcc without sse4.2/popcnt,
we use a plain C version to ensure inlining because
__builtin_popcount() and __builtin_popcountl() generate function
calls for that configuration. Our tests indicate this is still a
net win.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: Greg Burd <greg@burd.me>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
configure | 38 --------------------
configure.ac | 1 -
meson.build | 1 -
src/include/pg_config.h.in | 3 --
src/include/port/pg_bitutils.h | 59 +++++++++++++++++-------------
src/port/pg_bitutils.c | 65 ++--------------------------------
src/port/pg_popcount_aarch64.c | 23 +++---------
src/port/pg_popcount_x86.c | 43 +---------------------
8 files changed, 41 insertions(+), 192 deletions(-)
diff --git a/configure b/configure
index ba293931878..623aa397fae 100755
--- a/configure
+++ b/configure
@@ -15920,44 +15920,6 @@ cat >>confdefs.h <<_ACEOF
#define HAVE__BUILTIN_CTZ 1
_ACEOF
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __builtin_popcount" >&5
-$as_echo_n "checking for __builtin_popcount... " >&6; }
-if ${pgac_cv__builtin_popcount+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
-int
-call__builtin_popcount(unsigned int x)
-{
- return __builtin_popcount(x);
-}
-int
-main ()
-{
-
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- pgac_cv__builtin_popcount=yes
-else
- pgac_cv__builtin_popcount=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv__builtin_popcount" >&5
-$as_echo "$pgac_cv__builtin_popcount" >&6; }
-if test x"${pgac_cv__builtin_popcount}" = xyes ; then
-
-cat >>confdefs.h <<_ACEOF
-#define HAVE__BUILTIN_POPCOUNT 1
-_ACEOF
-
fi
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
diff --git a/configure.ac b/configure.ac
index 412fe358a2f..04c6a75bff7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1853,7 +1853,6 @@ PGAC_CHECK_BUILTIN_FUNC([__builtin_bswap64], [long int x])
# We assume that we needn't test all widths of these explicitly:
PGAC_CHECK_BUILTIN_FUNC([__builtin_clz], [unsigned int x])
PGAC_CHECK_BUILTIN_FUNC([__builtin_ctz], [unsigned int x])
-PGAC_CHECK_BUILTIN_FUNC([__builtin_popcount], [unsigned int x])
# __builtin_frame_address may draw a diagnostic for non-constant argument,
# so it needs a different test function.
PGAC_CHECK_BUILTIN_FUNC_PTR([__builtin_frame_address], [0])
diff --git a/meson.build b/meson.build
index 0722b16927e..c607d8ac69a 100644
--- a/meson.build
+++ b/meson.build
@@ -2004,7 +2004,6 @@ builtins = [
'ctz',
'constant_p',
'frame_address',
- 'popcount',
'unreachable',
]
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index c089f2252c3..301328b8cd3 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -530,9 +530,6 @@
/* Define to 1 if your compiler understands __builtin_$op_overflow. */
#undef HAVE__BUILTIN_OP_OVERFLOW
-/* Define to 1 if your compiler understands __builtin_popcount. */
-#undef HAVE__BUILTIN_POPCOUNT
-
/* Define to 1 if your compiler understands __builtin_types_compatible_p. */
#undef HAVE__BUILTIN_TYPES_COMPATIBLE_P
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 20c11b79c61..c9b1f5f17dc 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -276,47 +276,56 @@ pg_ceil_log2_64(uint64 num)
return pg_leftmost_one_pos64(num - 1) + 1;
}
-extern int pg_popcount32_portable(uint32 word);
-extern int pg_popcount64_portable(uint64 word);
extern uint64 pg_popcount_portable(const char *buf, int bytes);
extern uint64 pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask);
-#ifdef HAVE_X86_64_POPCNTQ
+#if defined(HAVE_X86_64_POPCNTQ) || defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
/*
- * Attempt to use SSE4.2 or AVX-512 instructions, but perform a runtime check
+ * Attempt to use specialized CPU instructions, but perform a runtime check
* first.
*/
-extern PGDLLIMPORT int (*pg_popcount32) (uint32 word);
-extern PGDLLIMPORT int (*pg_popcount64) (uint64 word);
extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
-#elif defined(USE_NEON)
-/* Use the Neon version of pg_popcount{32,64} without function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-
-/*
- * We can try to use an SVE-optimized pg_popcount() on some systems For that,
- * we do use a function pointer.
- */
-#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
-extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
-extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
-#else
-extern uint64 pg_popcount_optimized(const char *buf, int bytes);
-extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
-#endif
-
#else
/* Use a portable implementation -- no need for a function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
extern uint64 pg_popcount_optimized(const char *buf, int bytes);
extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
#endif
+/*
+ * pg_popcount32
+ * Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ */
+static inline int
+pg_popcount32(uint32 word)
+{
+ word -= (word >> 1) & 0x55555555;
+ word = (word & 0x33333333) + ((word >> 2) & 0x33333333);
+ return (((word + (word >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
+}
+
+/*
+ * pg_popcount64
+ * Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ */
+static inline int
+pg_popcount64(uint64 word)
+{
+ word -= (word >> 1) & UINT64CONST(0x5555555555555555);
+ word = (word & UINT64CONST(0x3333333333333333)) +
+ ((word >> 2) & UINT64CONST(0x3333333333333333));
+ word = (word + (word >> 4)) & UINT64CONST(0xf0f0f0f0f0f0f0f);
+ return (word * UINT64CONST(0x101010101010101)) >> 56;
+}
+
/*
* Returns the number of 1-bits in buf.
*
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c
index bec06c06fc3..49b130f1306 100644
--- a/src/port/pg_bitutils.c
+++ b/src/port/pg_bitutils.c
@@ -96,56 +96,6 @@ const uint8 pg_number_of_ones[256] = {
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
};
-/*
- * pg_popcount32_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount32_portable(uint32 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
-/*
- * pg_popcount64_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount64_portable(uint64 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
/*
* pg_popcount_portable
* Returns the number of 1-bits in buf
@@ -163,7 +113,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++);
+ popcnt += pg_popcount64(*words++);
bytes -= 8;
}
@@ -197,7 +147,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++ & maskv);
+ popcnt += pg_popcount64(*words++ & maskv);
bytes -= 8;
}
@@ -220,17 +170,6 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
* actual external functions. The compiler should be able to inline the
* portable versions here.
*/
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount32_portable(word);
-}
-
-int
-pg_popcount64(uint64 word)
-{
- return pg_popcount64_portable(word);
-}
/*
* pg_popcount_optimized
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index ba57f2cd4bd..357e938549f 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -291,28 +291,13 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
#endif /* ! USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32
- * Return number of 1 bits in word
- */
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount64((uint64) word);
-}
-
/*
* pg_popcount64
* Return number of 1 bits in word
*/
-int
-pg_popcount64(uint64 word)
+static inline int
+pg_popcount64_neon(uint64 word)
{
- /*
- * For some compilers, __builtin_popcountl() already emits Neon
- * instructions. The line below should compile to the same code on those
- * systems.
- */
return vaddv_u8(vcnt_u8(vld1_u8((const uint8 *) &word)));
}
@@ -383,7 +368,7 @@ pg_popcount_neon(const char *buf, int bytes)
*/
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
{
- popcnt += pg_popcount64(*((const uint64 *) buf));
+ popcnt += pg_popcount64_neon(*((const uint64 *) buf));
buf += sizeof(uint64);
}
@@ -465,7 +450,7 @@ pg_popcount_masked_neon(const char *buf, int bytes, bits8 mask)
*/
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
{
- popcnt += pg_popcount64(*((const uint64 *) buf) & mask64);
+ popcnt += pg_popcount64_neon(*((const uint64 *) buf) & mask64);
buf += sizeof(uint64);
}
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.c
index 7aebf69898b..6bce089432f 100644
--- a/src/port/pg_popcount_x86.c
+++ b/src/port/pg_popcount_x86.c
@@ -36,8 +36,6 @@
* operation, but in practice this is close enough, and "sse42" seems easier to
* follow than "popcnt" for these names.
*/
-static inline int pg_popcount32_sse42(uint32 word);
-static inline int pg_popcount64_sse42(uint64 word);
static uint64 pg_popcount_sse42(const char *buf, int bytes);
static uint64 pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask);
@@ -55,12 +53,8 @@ static uint64 pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask);
* what the current CPU supports) and then will call the pointer to fulfill the
* caller's request.
*/
-static int pg_popcount32_choose(uint32 word);
-static int pg_popcount64_choose(uint64 word);
static uint64 pg_popcount_choose(const char *buf, int bytes);
static uint64 pg_popcount_masked_choose(const char *buf, int bytes, bits8 mask);
-int (*pg_popcount32) (uint32 word) = pg_popcount32_choose;
-int (*pg_popcount64) (uint64 word) = pg_popcount64_choose;
uint64 (*pg_popcount_optimized) (const char *buf, int bytes) = pg_popcount_choose;
uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask) = pg_popcount_masked_choose;
@@ -157,7 +151,7 @@ pg_popcount_avx512_available(void)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
/*
- * These functions get called on the first call to pg_popcount32 etc.
+ * These functions get called on the first call to pg_popcount(), etc.
* They detect whether we can use the asm implementations, and replace
* the function pointers so that subsequent calls are routed directly to
* the chosen implementation.
@@ -167,15 +161,11 @@ choose_popcount_functions(void)
{
if (pg_popcount_sse42_available())
{
- pg_popcount32 = pg_popcount32_sse42;
- pg_popcount64 = pg_popcount64_sse42;
pg_popcount_optimized = pg_popcount_sse42;
pg_popcount_masked_optimized = pg_popcount_masked_sse42;
}
else
{
- pg_popcount32 = pg_popcount32_portable;
- pg_popcount64 = pg_popcount64_portable;
pg_popcount_optimized = pg_popcount_portable;
pg_popcount_masked_optimized = pg_popcount_masked_portable;
}
@@ -189,20 +179,6 @@ choose_popcount_functions(void)
#endif
}
-static int
-pg_popcount32_choose(uint32 word)
-{
- choose_popcount_functions();
- return pg_popcount32(word);
-}
-
-static int
-pg_popcount64_choose(uint64 word)
-{
- choose_popcount_functions();
- return pg_popcount64(word);
-}
-
static uint64
pg_popcount_choose(const char *buf, int bytes)
{
@@ -338,23 +314,6 @@ pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32_sse42
- * Return the number of 1 bits set in word
- */
-static inline int
-pg_popcount32_sse42(uint32 word)
-{
-#ifdef _MSC_VER
- return __popcnt(word);
-#else
- uint32 res;
-
-__asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc");
- return (int) res;
-#endif
-}
-
/*
* pg_popcount64_sse42
* Return the number of 1 bits set in word
--
2.50.1 (Apple Git-155)
--rlwGoOKBy5dABfM4
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v10-0003-Make-use-of-pg_popcount-in-more-places.patch
^ permalink raw reply [nested|flat] 19+ messages in thread
* [PATCH v11 2/4] Remove specialized word-length popcount implementations.
@ 2026-01-23 23:31 Nathan Bossart <nathan@postgresql.org>
0 siblings, 0 replies; 19+ messages in thread
From: Nathan Bossart @ 2026-01-23 23:31 UTC (permalink / raw)
The uses of these functions do not justify the level of
micro-optimization we've done and may even hurt performance in some
cases (e.g., due to using function pointers). This commit removes
all architecture-specific implementations of pg_popcount{32,64}()
and converts the portable ones to inlined functions in
pg_bitutils.h. These inlined versions should produce the same code
as before (but inlined), so in theory this is a net gain for many
machines. As an exception, for x86-64/gcc without sse4.2/popcnt,
we use a plain C version to ensure inlining because
__builtin_popcount() and __builtin_popcountl() generate function
calls for that configuration. Our tests indicate this is still a
net win.
Suggested-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: John Naylor <johncnaylorls@gmail.com>
Reviewed-by: Greg Burd <greg@burd.me>
Discussion: https://postgr.es/m/CANWCAZY7R%2Biy%2Br9YM_sySNydHzNqUirx1xk0tB3ej5HO62GdgQ%40mail.gmail.com
---
src/include/port/pg_bitutils.h | 72 ++++++++++++++++++++++------------
src/port/pg_bitutils.c | 65 +-----------------------------
src/port/pg_popcount_aarch64.c | 20 +++-------
src/port/pg_popcount_x86.c | 43 +-------------------
4 files changed, 56 insertions(+), 144 deletions(-)
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 20c11b79c61..3c58f6c6864 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -276,46 +276,70 @@ pg_ceil_log2_64(uint64 num)
return pg_leftmost_one_pos64(num - 1) + 1;
}
-extern int pg_popcount32_portable(uint32 word);
-extern int pg_popcount64_portable(uint64 word);
extern uint64 pg_popcount_portable(const char *buf, int bytes);
extern uint64 pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask);
-#ifdef HAVE_X86_64_POPCNTQ
+#if defined(HAVE_X86_64_POPCNTQ) || defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
/*
- * Attempt to use SSE4.2 or AVX-512 instructions, but perform a runtime check
+ * Attempt to use specialized CPU instructions, but perform a runtime check
* first.
*/
-extern PGDLLIMPORT int (*pg_popcount32) (uint32 word);
-extern PGDLLIMPORT int (*pg_popcount64) (uint64 word);
extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
-#elif defined(USE_NEON)
-/* Use the Neon version of pg_popcount{32,64} without function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-
-/*
- * We can try to use an SVE-optimized pg_popcount() on some systems For that,
- * we do use a function pointer.
- */
-#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
-extern PGDLLIMPORT uint64 (*pg_popcount_optimized) (const char *buf, int bytes);
-extern PGDLLIMPORT uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask);
#else
+/* Use a portable implementation -- no need for a function pointer. */
extern uint64 pg_popcount_optimized(const char *buf, int bytes);
extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+
#endif
-#else
-/* Use a portable implementation -- no need for a function pointer. */
-extern int pg_popcount32(uint32 word);
-extern int pg_popcount64(uint64 word);
-extern uint64 pg_popcount_optimized(const char *buf, int bytes);
-extern uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask);
+/*
+ * pg_popcount32
+ * Return the number of 1 bits set in word
+ *
+ * Adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ */
+static inline int
+pg_popcount32(uint32 word)
+{
+ word -= (word >> 1) & 0x55555555;
+ word = (word & 0x33333333) + ((word >> 2) & 0x33333333);
+ return (((word + (word >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
+}
+/*
+ * pg_popcount64
+ * Return the number of 1 bits set in word
+ *
+ * Plain C version adapted from
+ * https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel.
+ */
+static inline int
+pg_popcount64(uint64 word)
+{
+ /*
+ * On x86, gcc generates a function call for this built-in unless the
+ * popcnt instruction is available, so we use the plain C version in that
+ * case to ensure inlining.
+ */
+#if defined(HAVE__BUILTIN_POPCOUNT) && (defined(__POPCNT__) || !defined(__x86_64__))
+#if SIZEOF_LONG == 8
+ return __builtin_popcountl(word);
+#elif SIZEOF_LONG_LONG == 8
+ return __builtin_popcountll(word);
+#else
+#error "cannot find integer of the same size as uint64_t"
#endif
+#else
+ word -= (word >> 1) & UINT64CONST(0x5555555555555555);
+ word = (word & UINT64CONST(0x3333333333333333)) +
+ ((word >> 2) & UINT64CONST(0x3333333333333333));
+ word = (word + (word >> 4)) & UINT64CONST(0xf0f0f0f0f0f0f0f);
+ return (word * UINT64CONST(0x101010101010101)) >> 56;
+#endif
+}
/*
* Returns the number of 1-bits in buf.
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c
index bec06c06fc3..49b130f1306 100644
--- a/src/port/pg_bitutils.c
+++ b/src/port/pg_bitutils.c
@@ -96,56 +96,6 @@ const uint8 pg_number_of_ones[256] = {
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
};
-/*
- * pg_popcount32_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount32_portable(uint32 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
- return __builtin_popcount(word);
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
-/*
- * pg_popcount64_portable
- * Return the number of 1 bits set in word
- */
-int
-pg_popcount64_portable(uint64 word)
-{
-#ifdef HAVE__BUILTIN_POPCOUNT
-#if SIZEOF_LONG == 8
- return __builtin_popcountl(word);
-#elif SIZEOF_LONG_LONG == 8
- return __builtin_popcountll(word);
-#else
-#error "cannot find integer of the same size as uint64_t"
-#endif
-#else /* !HAVE__BUILTIN_POPCOUNT */
- int result = 0;
-
- while (word != 0)
- {
- result += pg_number_of_ones[word & 255];
- word >>= 8;
- }
-
- return result;
-#endif /* HAVE__BUILTIN_POPCOUNT */
-}
-
/*
* pg_popcount_portable
* Returns the number of 1-bits in buf
@@ -163,7 +113,7 @@ pg_popcount_portable(const char *buf, int bytes)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++);
+ popcnt += pg_popcount64(*words++);
bytes -= 8;
}
@@ -197,7 +147,7 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
while (bytes >= 8)
{
- popcnt += pg_popcount64_portable(*words++ & maskv);
+ popcnt += pg_popcount64(*words++ & maskv);
bytes -= 8;
}
@@ -220,17 +170,6 @@ pg_popcount_masked_portable(const char *buf, int bytes, bits8 mask)
* actual external functions. The compiler should be able to inline the
* portable versions here.
*/
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount32_portable(word);
-}
-
-int
-pg_popcount64(uint64 word)
-{
- return pg_popcount64_portable(word);
-}
/*
* pg_popcount_optimized
diff --git a/src/port/pg_popcount_aarch64.c b/src/port/pg_popcount_aarch64.c
index ba57f2cd4bd..f474ef45510 100644
--- a/src/port/pg_popcount_aarch64.c
+++ b/src/port/pg_popcount_aarch64.c
@@ -292,21 +292,11 @@ pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
#endif /* ! USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
/*
- * pg_popcount32
+ * pg_popcount64_neon
* Return number of 1 bits in word
*/
-int
-pg_popcount32(uint32 word)
-{
- return pg_popcount64((uint64) word);
-}
-
-/*
- * pg_popcount64
- * Return number of 1 bits in word
- */
-int
-pg_popcount64(uint64 word)
+static inline int
+pg_popcount64_neon(uint64 word)
{
/*
* For some compilers, __builtin_popcountl() already emits Neon
@@ -383,7 +373,7 @@ pg_popcount_neon(const char *buf, int bytes)
*/
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
{
- popcnt += pg_popcount64(*((const uint64 *) buf));
+ popcnt += pg_popcount64_neon(*((const uint64 *) buf));
buf += sizeof(uint64);
}
@@ -465,7 +455,7 @@ pg_popcount_masked_neon(const char *buf, int bytes, bits8 mask)
*/
for (; bytes >= sizeof(uint64); bytes -= sizeof(uint64))
{
- popcnt += pg_popcount64(*((const uint64 *) buf) & mask64);
+ popcnt += pg_popcount64_neon(*((const uint64 *) buf) & mask64);
buf += sizeof(uint64);
}
diff --git a/src/port/pg_popcount_x86.c b/src/port/pg_popcount_x86.c
index 7aebf69898b..6bce089432f 100644
--- a/src/port/pg_popcount_x86.c
+++ b/src/port/pg_popcount_x86.c
@@ -36,8 +36,6 @@
* operation, but in practice this is close enough, and "sse42" seems easier to
* follow than "popcnt" for these names.
*/
-static inline int pg_popcount32_sse42(uint32 word);
-static inline int pg_popcount64_sse42(uint64 word);
static uint64 pg_popcount_sse42(const char *buf, int bytes);
static uint64 pg_popcount_masked_sse42(const char *buf, int bytes, bits8 mask);
@@ -55,12 +53,8 @@ static uint64 pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask);
* what the current CPU supports) and then will call the pointer to fulfill the
* caller's request.
*/
-static int pg_popcount32_choose(uint32 word);
-static int pg_popcount64_choose(uint64 word);
static uint64 pg_popcount_choose(const char *buf, int bytes);
static uint64 pg_popcount_masked_choose(const char *buf, int bytes, bits8 mask);
-int (*pg_popcount32) (uint32 word) = pg_popcount32_choose;
-int (*pg_popcount64) (uint64 word) = pg_popcount64_choose;
uint64 (*pg_popcount_optimized) (const char *buf, int bytes) = pg_popcount_choose;
uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask) = pg_popcount_masked_choose;
@@ -157,7 +151,7 @@ pg_popcount_avx512_available(void)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
/*
- * These functions get called on the first call to pg_popcount32 etc.
+ * These functions get called on the first call to pg_popcount(), etc.
* They detect whether we can use the asm implementations, and replace
* the function pointers so that subsequent calls are routed directly to
* the chosen implementation.
@@ -167,15 +161,11 @@ choose_popcount_functions(void)
{
if (pg_popcount_sse42_available())
{
- pg_popcount32 = pg_popcount32_sse42;
- pg_popcount64 = pg_popcount64_sse42;
pg_popcount_optimized = pg_popcount_sse42;
pg_popcount_masked_optimized = pg_popcount_masked_sse42;
}
else
{
- pg_popcount32 = pg_popcount32_portable;
- pg_popcount64 = pg_popcount64_portable;
pg_popcount_optimized = pg_popcount_portable;
pg_popcount_masked_optimized = pg_popcount_masked_portable;
}
@@ -189,20 +179,6 @@ choose_popcount_functions(void)
#endif
}
-static int
-pg_popcount32_choose(uint32 word)
-{
- choose_popcount_functions();
- return pg_popcount32(word);
-}
-
-static int
-pg_popcount64_choose(uint64 word)
-{
- choose_popcount_functions();
- return pg_popcount64(word);
-}
-
static uint64
pg_popcount_choose(const char *buf, int bytes)
{
@@ -338,23 +314,6 @@ pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask)
#endif /* USE_AVX512_POPCNT_WITH_RUNTIME_CHECK */
-/*
- * pg_popcount32_sse42
- * Return the number of 1 bits set in word
- */
-static inline int
-pg_popcount32_sse42(uint32 word)
-{
-#ifdef _MSC_VER
- return __popcnt(word);
-#else
- uint32 res;
-
-__asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc");
- return (int) res;
-#endif
-}
-
/*
* pg_popcount64_sse42
* Return the number of 1 bits set in word
--
2.50.1 (Apple Git-155)
--mih0IfQp6StcW7xc
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename=v11-0003-Make-use-of-pg_popcount-in-more-places.patch
^ permalink raw reply [nested|flat] 19+ messages in thread
* Enforce INSERT RLS checks for FOR PORTION OF leftovers?
@ 2026-05-01 18:53 Ayush Tiwari <ayushtiwari.slg01@gmail.com>
0 siblings, 1 reply; 19+ messages in thread
From: Ayush Tiwari @ 2026-05-01 18:53 UTC (permalink / raw)
To: pgsql-hackers; Peter Eisentraut <peter@eisentraut.org>
Hi,
I found what looks like a discrepancy where UPDATE/DELETE FOR
PORTION OF commands bypass INSERT RLS WITH CHECK
policies when inserting temporal leftover rows. Not sure if it's already
flagged (could not find it in DL).
While it is intentional that ExecForPortionOfLeftovers() skips INSERT ACL
permission checks, the leftover rows are newly inserted rows and should
still satisfy INSERT/ALL RLS policies unless I'm missing something.
Currently, the rewrite phase only
attaches UPDATE/DELETE RLS checks for the target relation, leaving
ExecInsert() without a WCO_RLS_INSERT_CHECK to enforce for the
leftovers.
Maybe we should address this in rowsecurity.c by fetching CMD_INSERT
policies and adding them as WCO_RLS_INSERT_CHECK entries for queries
with a FOR PORTION OF clause?
Something like this:
--- a/src/backend/rewrite/rowsecurity.c
+++ b/src/backend/rewrite/rowsecurity.c
@@ -393,6 +393,34 @@ get_row_security_policies(Query *root, RangeTblEntry
*rte, int rt_index,
}
}
+ /*
+ * UPDATE/DELETE FOR PORTION OF commands insert temporal leftovers via
+ * ExecInsert(). Those internal inserts intentionally skip INSERT ACL
+ * permission checks, but they still create new rows and must satisfy any
+ * INSERT/ALL RLS WITH CHECK policies.
+ */
+ if ((commandType == CMD_UPDATE || commandType == CMD_DELETE) &&
+ root->forPortionOf != NULL)
+ {
+ List *insert_permissive_policies;
+ List *insert_restrictive_policies;
+
+ /* This should be the target relation */
+ Assert(rt_index == root->resultRelation);
+
+ get_policies_for_relation(rel, CMD_INSERT, user_id,
+ &insert_permissive_policies,
+ &insert_restrictive_policies);
+
+ add_with_check_options(rel, rt_index,
+ WCO_RLS_INSERT_CHECK,
+ insert_permissive_policies,
+ insert_restrictive_policies,
+ withCheckOptions,
+ hasSubLinks,
+ false);
+ }
+
Thoughts?
Regards,
Ayush
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Enforce INSERT RLS checks for FOR PORTION OF leftovers?
@ 2026-05-04 11:29 Ayush Tiwari <ayushtiwari.slg01@gmail.com>
parent: Ayush Tiwari <ayushtiwari.slg01@gmail.com>
0 siblings, 1 reply; 19+ messages in thread
From: Ayush Tiwari @ 2026-05-04 11:29 UTC (permalink / raw)
To: pgsql-hackers; Peter Eisentraut <peter@eisentraut.org>; Paul A Jungwirth <pj@illuminatedcomputing.com>
Hi,
On Sat, 2 May 2026 at 00:23, Ayush Tiwari <ayushtiwari.slg01@gmail.com>
wrote:
> Hi,
>
> I found what looks like a discrepancy where UPDATE/DELETE FOR
> PORTION OF commands bypass INSERT RLS WITH CHECK
> policies when inserting temporal leftover rows. Not sure if it's already
> flagged (could not find it in DL).
>
> While it is intentional that ExecForPortionOfLeftovers() skips INSERT ACL
> permission checks, the leftover rows are newly inserted rows and should
> still satisfy INSERT/ALL RLS policies unless I'm missing something.
>
>
Sharing a SQL repro example:
CREATE ROLE u;
CREATE TABLE t (id int, valid_at daterange NOT NULL, name text);
ALTER TABLE t ENABLE ROW LEVEL SECURITY;
CREATE POLICY p_all ON t FOR ALL TO u USING (true) WITH CHECK (true);
CREATE POLICY p_ins ON t FOR INSERT TO u WITH CHECK (false);
GRANT SELECT, INSERT, UPDATE, DELETE ON t TO u;
INSERT INTO t VALUES (1, daterange('2018-01-01','2020-01-01'), 'ok');
SET ROLE u;
-- (A) Fails as expected: new row violates row-level security policy
INSERT INTO t VALUES (2, daterange('2018-01-01','2020-01-01'), 'ok');
-- (B) Should fail the same way (creates leftover rows), but silently
succeeds
UPDATE t FOR PORTION OF valid_at FROM '2019-01-01' TO '2019-06-01'
SET name = 'ok' WHERE id = 1;
If this is expected we need to change the documentation of policy
and if it is not, should we go with something like I shared in
upthread, I can send a patch file if required.
Regards,
Ayush
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Enforce INSERT RLS checks for FOR PORTION OF leftovers?
@ 2026-05-16 16:50 Paul A Jungwirth <pj@illuminatedcomputing.com>
parent: Ayush Tiwari <ayushtiwari.slg01@gmail.com>
0 siblings, 1 reply; 19+ messages in thread
From: Paul A Jungwirth @ 2026-05-16 16:50 UTC (permalink / raw)
To: Ayush Tiwari <ayushtiwari.slg01@gmail.com>; +Cc: pgsql-hackers; Peter Eisentraut <peter@eisentraut.org>
On Mon, May 4, 2026 at 4:29 AM Ayush Tiwari <ayushtiwari.slg01@gmail.com> wrote:
>
> On Sat, 2 May 2026 at 00:23, Ayush Tiwari <ayushtiwari.slg01@gmail.com> wrote:
>>
>> I found what looks like a discrepancy where UPDATE/DELETE FOR
>> PORTION OF commands bypass INSERT RLS WITH CHECK
>> policies when inserting temporal leftover rows. Not sure if it's already
>> flagged (could not find it in DL).
>>
>> While it is intentional that ExecForPortionOfLeftovers() skips INSERT ACL
>> permission checks, the leftover rows are newly inserted rows and should
>> still satisfy INSERT/ALL RLS policies unless I'm missing something.
>
> Sharing a SQL repro example:
>
> CREATE ROLE u;
> CREATE TABLE t (id int, valid_at daterange NOT NULL, name text);
> ALTER TABLE t ENABLE ROW LEVEL SECURITY;
> CREATE POLICY p_all ON t FOR ALL TO u USING (true) WITH CHECK (true);
> CREATE POLICY p_ins ON t FOR INSERT TO u WITH CHECK (false);
> GRANT SELECT, INSERT, UPDATE, DELETE ON t TO u;
> INSERT INTO t VALUES (1, daterange('2018-01-01','2020-01-01'), 'ok');
>
> SET ROLE u;
>
> -- (A) Fails as expected: new row violates row-level security policy
> INSERT INTO t VALUES (2, daterange('2018-01-01','2020-01-01'), 'ok');
>
> -- (B) Should fail the same way (creates leftover rows), but silently succeeds
> UPDATE t FOR PORTION OF valid_at FROM '2019-01-01' TO '2019-06-01'
> SET name = 'ok' WHERE id = 1;
>
> If this is expected we need to change the documentation of policy
> and if it is not, should we go with something like I shared in
> upthread, I can send a patch file if required.
Skipping the RLS checks to insert the leftovers seems like the correct
behavior to me, since we are skipping the ACL checks (per the
standard). Shouldn't it be consistent?
I think the reason we skip the checks is that semantically, the
leftovers aren't changing anything: they are preserving the history
that is already there.
I'm happy to write a doc patch to make this explicit. Does anyone
disagree about the correct behavior though?
Yours,
--
Paul ~{:-)
pj@illuminatedcomputing.com
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Enforce INSERT RLS checks for FOR PORTION OF leftovers?
@ 2026-06-05 16:46 Ayush Tiwari <ayushtiwari.slg01@gmail.com>
parent: Paul A Jungwirth <pj@illuminatedcomputing.com>
0 siblings, 1 reply; 19+ messages in thread
From: Ayush Tiwari @ 2026-06-05 16:46 UTC (permalink / raw)
To: Paul A Jungwirth <pj@illuminatedcomputing.com>; +Cc: pgsql-hackers; Peter Eisentraut <peter@eisentraut.org>
Hi,
On Sat, 16 May 2026 at 22:20, Paul A Jungwirth <pj@illuminatedcomputing.com>
wrote:
Skipping the RLS checks to insert the leftovers seems like the correct
> behavior to me, since we are skipping the ACL checks (per the
> standard). Shouldn't it be consistent?
>
> I think the reason we skip the checks is that semantically, the
> leftovers aren't changing anything: they are preserving the history
> that is already there.
>
> I'm happy to write a doc patch to make this explicit. Does anyone
> disagree about the correct behavior though?
>
I'd treat them separately. Skipping INSERT ACL is a standard-driven
call about command privilege; RLS WITH CHECK is a per-row predicate
on the data, and the leftover really is a new heap tuple written by
the current role.
With `WITH CHECK (name <> 'denied')` and a pre-existing 'denied' row
(loaded by COPY or before the policy), UPDATE FOR PORTION OF that
touches only the range column writes leftovers the role could not
have INSERTed directly. That's the part that bothers me, especially
since RLS tends to be deployed as a hard security boundary rather
than something users expect to opt out of per statement type.
Please correct me if I'm wrong here.
Regards,
Ayush
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Enforce INSERT RLS checks for FOR PORTION OF leftovers?
@ 2026-07-15 15:35 Nathan Bossart <nathandbossart@gmail.com>
parent: Ayush Tiwari <ayushtiwari.slg01@gmail.com>
0 siblings, 1 reply; 19+ messages in thread
From: Nathan Bossart @ 2026-07-15 15:35 UTC (permalink / raw)
To: Ayush Tiwari <ayushtiwari.slg01@gmail.com>; +Cc: Paul A Jungwirth <pj@illuminatedcomputing.com>; pgsql-hackers; Peter Eisentraut <peter@eisentraut.org>
[RMT hat]
This is listed as an open item for v19 [0], but there's been no update to
the thread since June 5th. Is it on track for resolution sometime soon?
[0] https://wiki.postgresql.org/wiki/PostgreSQL_19_Open_Items
--
nathan
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: Enforce INSERT RLS checks for FOR PORTION OF leftovers?
@ 2026-07-15 17:17 Paul A Jungwirth <pj@illuminatedcomputing.com>
parent: Nathan Bossart <nathandbossart@gmail.com>
0 siblings, 0 replies; 19+ messages in thread
From: Paul A Jungwirth @ 2026-07-15 17:17 UTC (permalink / raw)
To: Nathan Bossart <nathandbossart@gmail.com>; +Cc: Ayush Tiwari <ayushtiwari.slg01@gmail.com>; pgsql-hackers; Peter Eisentraut <peter@eisentraut.org>
On Wed, Jul 15, 2026 at 8:35 AM Nathan Bossart <nathandbossart@gmail.com> wrote:
>
> [RMT hat]
>
> This is listed as an open item for v19 [0], but there's been no update to
> the thread since June 5th. Is it on track for resolution sometime soon?
I think the patch at [1] is ready to go, but it's on a separate thread.
I updated the Open Items list in include both conversations.
[1] https://www.postgresql.org/message-id/20DF964A-C291-491A-9126-62753DCBD7CE%40gmail.com
Yours,
--
Paul ~{:-)
pj@illuminatedcomputing.com
^ permalink raw reply [nested|flat] 19+ messages in thread
end of thread, other threads:[~2026-07-15 17:17 UTC | newest]
Thread overview: 19+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-07-09 09:54 [PATCH 1/5] Make XLogReaderInvalReadState static. Antonin Houska <ah@cybertec.at>
2023-08-07 18:37 [PATCH v3] psql: Add tab-complete for optional view parameters Christoph Heiss <christoph@c8h4.io>
2023-08-07 18:37 [PATCH v3] psql: Add tab-complete for optional view parameters Christoph Heiss <christoph@c8h4.io>
2023-08-07 18:37 [PATCH v3] psql: Add tab-complete for optional view parameters Christoph Heiss <christoph@c8h4.io>
2024-10-16 18:21 [PATCH v4 2/8] Address space reservation for shared memory Dmitrii Dolgov <9erthalion6@gmail.com>
2026-01-23 23:31 [PATCH v9 2/3] Remove specialized word-length popcount implementations. Nathan Bossart <nathan@postgresql.org>
2026-01-23 23:31 [PATCH v10 2/3] Remove specialized word-length popcount implementations. Nathan Bossart <nathan@postgresql.org>
2026-01-23 23:31 [PATCH v13 2/5] Remove specialized word-length popcount implementations. Nathan Bossart <nathan@postgresql.org>
2026-01-23 23:31 [PATCH v8 2/3] Remove specialized word-length popcount implementations. Nathan Bossart <nathan@postgresql.org>
2026-01-23 23:31 [PATCH v6 3/3] Remove specialized word-length popcount implementations. Nathan Bossart <nathan@postgresql.org>
2026-01-23 23:31 [PATCH v12 2/4] Remove specialized word-length popcount implementations. Nathan Bossart <nathan@postgresql.org>
2026-01-23 23:31 [PATCH v11 2/4] Remove specialized word-length popcount implementations. Nathan Bossart <nathan@postgresql.org>
2026-01-23 23:31 [PATCH v5 3/3] Remove specialized word-length popcount implementations. Nathan Bossart <nathan@postgresql.org>
2026-05-01 18:53 Enforce INSERT RLS checks for FOR PORTION OF leftovers? Ayush Tiwari <ayushtiwari.slg01@gmail.com>
2026-05-04 11:29 ` Re: Enforce INSERT RLS checks for FOR PORTION OF leftovers? Ayush Tiwari <ayushtiwari.slg01@gmail.com>
2026-05-16 16:50 ` Re: Enforce INSERT RLS checks for FOR PORTION OF leftovers? Paul A Jungwirth <pj@illuminatedcomputing.com>
2026-06-05 16:46 ` Re: Enforce INSERT RLS checks for FOR PORTION OF leftovers? Ayush Tiwari <ayushtiwari.slg01@gmail.com>
2026-07-15 15:35 ` Re: Enforce INSERT RLS checks for FOR PORTION OF leftovers? Nathan Bossart <nathandbossart@gmail.com>
2026-07-15 17:17 ` Re: Enforce INSERT RLS checks for FOR PORTION OF leftovers? Paul A Jungwirth <pj@illuminatedcomputing.com>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox