public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 4/8] BRIN bloom indexes
6+ messages / 3 participants
[nested] [flat]
* [PATCH 4/8] BRIN bloom indexes
@ 2020-04-02 00:57 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 6+ messages in thread
From: Tomas Vondra @ 2020-04-02 00:57 UTC (permalink / raw)
---
doc/src/sgml/brin.sgml | 215 +++++
doc/src/sgml/ref/create_index.sgml | 31 +
src/backend/access/brin/Makefile | 1 +
src/backend/access/brin/brin_bloom.c | 980 +++++++++++++++++++++++
src/include/access/brin.h | 2 +
src/include/access/brin_internal.h | 4 +
src/include/catalog/pg_amop.dat | 165 ++++
src/include/catalog/pg_amproc.dat | 430 ++++++++++
src/include/catalog/pg_opclass.dat | 69 ++
src/include/catalog/pg_opfamily.dat | 36 +
src/include/catalog/pg_proc.dat | 20 +
src/test/regress/expected/brin_bloom.out | 456 +++++++++++
src/test/regress/expected/opr_sanity.out | 3 +-
src/test/regress/expected/psql.out | 3 +-
src/test/regress/parallel_schedule | 5 +
src/test/regress/serial_schedule | 1 +
src/test/regress/sql/brin_bloom.sql | 404 ++++++++++
17 files changed, 2823 insertions(+), 2 deletions(-)
create mode 100644 src/backend/access/brin/brin_bloom.c
create mode 100644 src/test/regress/expected/brin_bloom.out
create mode 100644 src/test/regress/sql/brin_bloom.sql
diff --git a/doc/src/sgml/brin.sgml b/doc/src/sgml/brin.sgml
index 55b6272db6..e10fc64d6b 100644
--- a/doc/src/sgml/brin.sgml
+++ b/doc/src/sgml/brin.sgml
@@ -132,6 +132,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
</row>
</thead>
<tbody>
+ <row>
+ <entry><literal>int8_bloom_ops</literal></entry>
+ <entry><type>bigint</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>int8_minmax_ops</literal></entry>
<entry><type>bigint</type></entry>
@@ -183,6 +190,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>|&></literal>
</entry>
</row>
+ <row>
+ <entry><literal>bytea_bloom_ops</literal></entry>
+ <entry><type>bytea</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>bytea_minmax_ops</literal></entry>
<entry><type>bytea</type></entry>
@@ -194,6 +208,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>bpchar_bloom_ops</literal></entry>
+ <entry><type>character</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>bpchar_minmax_ops</literal></entry>
<entry><type>character</type></entry>
@@ -205,6 +226,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>char_bloom_ops</literal></entry>
+ <entry><type>"char"</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>char_minmax_ops</literal></entry>
<entry><type>"char"</type></entry>
@@ -216,6 +244,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>date_bloom_ops</literal></entry>
+ <entry><type>date</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>date_minmax_ops</literal></entry>
<entry><type>date</type></entry>
@@ -227,6 +262,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>float8_bloom_ops</literal></entry>
+ <entry><type>double precision</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>float8_minmax_ops</literal></entry>
<entry><type>double precision</type></entry>
@@ -238,6 +280,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>inet_bloom_ops</literal></entry>
+ <entry><type>inet</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>inet_minmax_ops</literal></entry>
<entry><type>inet</type></entry>
@@ -261,6 +310,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal><<</literal>
</entry>
</row>
+ <row>
+ <entry><literal>int4_bloom_ops</literal></entry>
+ <entry><type>integer</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>int4_minmax_ops</literal></entry>
<entry><type>integer</type></entry>
@@ -272,6 +328,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>interval_bloom_ops</literal></entry>
+ <entry><type>interval</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>interval_minmax_ops</literal></entry>
<entry><type>interval</type></entry>
@@ -283,6 +346,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>macaddr_bloom_ops</literal></entry>
+ <entry><type>macaddr</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>macaddr_minmax_ops</literal></entry>
<entry><type>macaddr</type></entry>
@@ -294,6 +364,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>macaddr8_bloom_ops</literal></entry>
+ <entry><type>macaddr8</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>macaddr8_minmax_ops</literal></entry>
<entry><type>macaddr8</type></entry>
@@ -305,6 +382,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>name_bloom_ops</literal></entry>
+ <entry><type>name</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>name_minmax_ops</literal></entry>
<entry><type>name</type></entry>
@@ -316,6 +400,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>numeric_bloom_ops</literal></entry>
+ <entry><type>numeric</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>numeric_minmax_ops</literal></entry>
<entry><type>numeric</type></entry>
@@ -327,6 +418,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>pg_lsn_bloom_ops</literal></entry>
+ <entry><type>pg_lsn</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>pg_lsn_minmax_ops</literal></entry>
<entry><type>pg_lsn</type></entry>
@@ -338,6 +436,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>oid_bloom_ops</literal></entry>
+ <entry><type>oid</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>oid_minmax_ops</literal></entry>
<entry><type>oid</type></entry>
@@ -369,6 +474,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>>=</literal>
</entry>
</row>
+ <row>
+ <entry><literal>float4_bloom_ops</literal></entry>
+ <entry><type>real</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>float4_minmax_ops</literal></entry>
<entry><type>real</type></entry>
@@ -380,6 +492,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>int2_bloom_ops</literal></entry>
+ <entry><type>smallint</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>int2_minmax_ops</literal></entry>
<entry><type>smallint</type></entry>
@@ -391,6 +510,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>text_bloom_ops</literal></entry>
+ <entry><type>text</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>text_minmax_ops</literal></entry>
<entry><type>text</type></entry>
@@ -413,6 +539,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>timestamp_bloom_ops</literal></entry>
+ <entry><type>timestamp without time zone</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>timestamp_minmax_ops</literal></entry>
<entry><type>timestamp without time zone</type></entry>
@@ -424,6 +557,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>timestamptz_bloom_ops</literal></entry>
+ <entry><type>timestamp with time zone</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>timestamptz_minmax_ops</literal></entry>
<entry><type>timestamp with time zone</type></entry>
@@ -435,6 +575,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>time_bloom_ops</literal></entry>
+ <entry><type>time without time zone</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>time_minmax_ops</literal></entry>
<entry><type>time without time zone</type></entry>
@@ -446,6 +593,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>timetz_bloom_ops</literal></entry>
+ <entry><type>time with time zone</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>timetz_minmax_ops</literal></entry>
<entry><type>time with time zone</type></entry>
@@ -457,6 +611,13 @@ LOG: request for BRIN range summarization for index "brin_wi_idx" page 128 was
<literal>></literal>
</entry>
</row>
+ <row>
+ <entry><literal>uuid_bloom_ops</literal></entry>
+ <entry><type>uuid</type></entry>
+ <entry>
+ <literal>=</literal>
+ </entry>
+ </row>
<row>
<entry><literal>uuid_minmax_ops</literal></entry>
<entry><type>uuid</type></entry>
@@ -841,6 +1002,60 @@ typedef struct BrinOpcInfo
function can improve index performance.
</para>
+ <para>
+ To write an operator class for a data type that implements only an equality
+ operator and supports hashing, it is possible to use the bloom support procedures
+ alongside the corresponding operators, as shown in
+ <xref linkend="brin-extensibility-bloom-table"/>.
+ All operator class members (procedures and operators) are mandatory.
+ </para>
+
+ <table id="brin-extensibility-bloom-table">
+ <title>Procedure and Support Numbers for Bloom Operator Classes</title>
+ <tgroup cols="2">
+ <thead>
+ <row>
+ <entry>Operator class member</entry>
+ <entry>Object</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry>Support Procedure 1</entry>
+ <entry>internal function <function>brin_bloom_opcinfo()</function></entry>
+ </row>
+ <row>
+ <entry>Support Procedure 2</entry>
+ <entry>internal function <function>brin_bloom_add_value()</function></entry>
+ </row>
+ <row>
+ <entry>Support Procedure 3</entry>
+ <entry>internal function <function>brin_bloom_consistent()</function></entry>
+ </row>
+ <row>
+ <entry>Support Procedure 4</entry>
+ <entry>internal function <function>brin_bloom_union()</function></entry>
+ </row>
+ <row>
+ <entry>Support Procedure 11</entry>
+ <entry>function to compute hash of an element</entry>
+ </row>
+ <row>
+ <entry>Operator Strategy 1</entry>
+ <entry>operator equal-to</entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <para>
+ Support procedure numbers 1-10 are reserved for the BRIN internal
+ functions, so the SQL level functions start with number 11. Support
+ function number 11 is the main function required to build the index.
+ It should accept one argument with the same data type as the operator class,
+ and return a hash of the value.
+ </para>
+
<para>
Both minmax and inclusion operator classes support cross-data-type
operators, though with these the dependencies become more complicated.
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 33aa64e81d..9c90d451a7 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -555,6 +555,37 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
</para>
</listitem>
</varlistentry>
+
+ <varlistentry>
+ <term><literal>n_distinct_per_range</literal></term>
+ <listitem>
+ <para>
+ Defines the estimated number of distinct non-null values in the block
+ range, used by <acronym>BRIN</acronym> bloom indexes for sizing of the
+ Bloom filter. It behaves similarly to <literal>n_distinct</literal> option
+ for <xref linkend="sql-altertable"/>. When set to a positive value,
+ each block range is assumed to contain this number of distinct non-null
+ values. When set to a negative value, which must be greater than or
+ equal to -1, the number of distinct non-null is assumed linear with
+ the maximum possible number of tuples in the block range (about 290
+ rows per block). The default values is <literal>-0.1</literal>, and
+ the minimum number of distinct non-null values is <literal>128</literal>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><literal>false_positive_rate</literal></term>
+ <listitem>
+ <para>
+ Defines the desired false positive rate used by <acronym>BRIN</acronym>
+ bloom indexes for sizing of the Bloom filter. The values must be be
+ greater than 0.0 and smaller than 1.0. The default values is 0.01,
+ which is 1% false positive rate.
+ </para>
+ </listitem>
+ </varlistentry>
+
</variablelist>
</refsect2>
diff --git a/src/backend/access/brin/Makefile b/src/backend/access/brin/Makefile
index 468e1e289a..6b56131215 100644
--- a/src/backend/access/brin/Makefile
+++ b/src/backend/access/brin/Makefile
@@ -14,6 +14,7 @@ include $(top_builddir)/src/Makefile.global
OBJS = \
brin.o \
+ brin_bloom.o \
brin_inclusion.o \
brin_minmax.o \
brin_pageops.o \
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
new file mode 100644
index 0000000000..b119e4e264
--- /dev/null
+++ b/src/backend/access/brin/brin_bloom.c
@@ -0,0 +1,980 @@
+/*
+ * brin_bloom.c
+ * Implementation of Bloom opclass for BRIN
+ *
+ * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * A BRIN opclass summarizing page range into a bloom filter.
+ *
+ * Bloom filters allow efficient test whether a given page range contains
+ * a particular value. Therefore, if we summarize each page range into a
+ * bloom filter, we can easily and cheaply test wheter it containst values
+ * we get later.
+ *
+ * The index only supports equality operator, similarly to hash indexes.
+ * BRIN bloom indexes are however much smaller, and support only bitmap
+ * scans.
+ *
+ * Note: Don't confuse this with bloom indexes, implemented in a contrib
+ * module. That extension implements an entirely new AM, building a bloom
+ * filter on multiple columns in a single row. This opclass works with an
+ * existing AM (BRIN) and builds bloom filter on a column.
+ *
+ *
+ * values vs. hashes
+ * -----------------
+ *
+ * The original column values are not used directly, but are first hashed
+ * using the regular type-specific hash function, producing a uint32 hash.
+ * And this hash value is then added to the summary - either stored as is
+ * (in the sorted mode), or hashed again and added to the bloom filter.
+ *
+ * This allows the code to treat all data types (byval/byref/...) the same
+ * way, with only minimal space requirements. For example we don't need to
+ * store varlena types differently in the sorted mode, etc. Everything is
+ * uint32 making it much simpler.
+ *
+ * Of course, this assumes the built-in hash function is reasonably good,
+ * without too many collisions etc. But that does seem to be the case, at
+ * least based on past experience. After all, the same hash functions are
+ * used for hash indexes, hash partitioning and so on.
+ *
+ *
+ * sizing the bloom filter
+ * -----------------------
+ *
+ * Size of a bloom filter depends on the number of distinct values we will
+ * store in it, and the desired false positive rate. The higher the number
+ * of distinct values and/or the lower the false positive rate, the larger
+ * the bloom filter. On the other hand, we want to keep the index as small
+ * as possible - that's one of the basic advantages of BRIN indexes.
+ *
+ * The number of distinct elements (in a page range) depends on the data,
+ * we can consider it fixed. This simplifies the trade-off to just false
+ * positive rate vs. size.
+ *
+ * At the page range level, false positive rate is a probability the bloom
+ * filter matches a random value. For the whole index (with sufficiently
+ * many page ranges) it represents the fraction of the index ranges (and
+ * thus fraction of the table to be scanned) matching the random value.
+ *
+ * Furthermore, the size of the bloom filter is subject to implementation
+ * limits - it has to fit onto a single index page (8kB by default). As
+ * the bitmap is inherently random, compression can't reliably help here.
+ * To reduce the size of a filter (to fit to a page), we have to either
+ * accept higher false positive rate (undesirable), or reduce the number
+ * of distinct items to be stored in the filter. We can't quite the input
+ * data, of course, but we may make the BRIN page ranges smaller - instead
+ * of the default 128 pages (1MB) we may build index with 16-page ranges,
+ * or something like that. This does help even for random data sets, as
+ * the number of rows per heap page is limited (to ~290 with very narrow
+ * tables, likely ~20 in practice).
+ *
+ * Of course, good sizing decisions depend on having the necessary data,
+ * i.e. number of distinct values in a page range (of a given size) and
+ * table size (to estimate cost change due to change in false positive
+ * rate due to having larger index vs. scanning larger indexes). We may
+ * not have that data - for example when building an index on empty table
+ * it's not really possible. And for some data we only have estimates for
+ * the whole table and we can only estimate per-range values (ndistinct).
+ *
+ * Another challenge is that while the bloom filter is per-column, it's
+ * the whole index tuple that has to fit into a page. And for multi-column
+ * indexes that may include pieces we have no control over (not necessarily
+ * bloom filters, the other columns may use other BRIN opclasses). So it's
+ * not entirely clear how to distrubute the space between those columns.
+ *
+ * The current logic, implemented in brin_bloom_get_ndistinct, attempts to
+ * make some basic sizing decisions, based on the table ndistinct estimate.
+ *
+ *
+ * sort vs. hash
+ * -------------
+ *
+ * As explained in the preceding section, sizing bloom filters correctly is
+ * difficult in practice. It's also true that bloom filters quickly degrade
+ * after exceeding the expected number of distinct items. It's therefore
+ * expected that people will overshoot the parameters a bit (particularly
+ * the ndistinct per page range), perhaps by a factor of 2 or more.
+ *
+ * It's also possible the data set is not uniform - it may have ranges with
+ * very many distinct items per range, but also ranges with only very few
+ * distinct items. The bloom filter has to be sized for the more variable
+ * ranges, making it rather wasteful in the less variable part.
+ *
+ * For example, if some page ranges have 1000 distinct values, that means
+ * about ~1.2kB bloom filter with 1% false positive rate. For page ranges
+ * that only contain 10 distinct values, that's wasteful, because we might
+ * store the values (the uint32 hashes) in just 40B.
+ *
+ * To address these issues, the opclass stores the raw values directly, and
+ * only switches to the actual bloom filter after reaching the same space
+ * requirements.
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/access/brin/brin_bloom.c
+ */
+#include "postgres.h"
+
+#include "access/genam.h"
+#include "access/brin.h"
+#include "access/brin_internal.h"
+#include "access/brin_tuple.h"
+#include "access/hash.h"
+#include "access/htup_details.h"
+#include "access/reloptions.h"
+#include "access/stratnum.h"
+#include "catalog/pg_type.h"
+#include "catalog/pg_amop.h"
+#include "utils/builtins.h"
+#include "utils/datum.h"
+#include "utils/lsyscache.h"
+#include "utils/rel.h"
+#include "utils/syscache.h"
+
+#include <math.h>
+
+#define BloomEqualStrategyNumber 1
+
+/*
+ * Additional SQL level support functions
+ *
+ * Procedure numbers must not use values reserved for BRIN itself; see
+ * brin_internal.h.
+ */
+#define BLOOM_MAX_PROCNUMS 1 /* maximum support procs we need */
+#define PROCNUM_HASH 11 /* required */
+
+/*
+ * Subtract this from procnum to obtain index in BloomOpaque arrays
+ * (Must be equal to minimum of private procnums).
+ */
+#define PROCNUM_BASE 11
+
+/*
+ * Flags. We only use a single bit for now, to decide whether we're in the
+ * sorted or hash phase. So we have 15 bits for future use, if needed. The
+ * filters are expected to be hundreds of bytes, so this is negligible.
+ */
+#define BLOOM_FLAG_PHASE_HASH 0x0001
+
+#define BLOOM_IS_HASHED(f) ((f)->flags & BLOOM_FLAG_PHASE_HASH)
+#define BLOOM_IS_SORTED(f) (!BLOOM_IS_HASHED(f))
+
+/*
+ * Number of hashes to accumulate before deduplicating and sorting in the
+ * sort phase? We want this fairly small to reduce the amount of space and
+ * also speed-up bsearch lookups.
+ */
+#define BLOOM_MAX_UNSORTED 32
+
+/*
+ * Storage type for BRIN's reloptions.
+ */
+typedef struct BloomOptions
+{
+ int32 vl_len_; /* varlena header (do not touch directly!) */
+ double nDistinctPerRange; /* number of distinct values per range */
+ double falsePositiveRate; /* false positive for bloom filter */
+} BloomOptions;
+
+/*
+ * The current min value (16) is somewhat arbitrary, but it's based
+ * on the fact that the filter header is ~20B alone, which is about
+ * the same as the filter bitmap for 16 distinct items with 1% false
+ * positive rate. So by allowing lower values we'd not gain much. In
+ * any case, the min should not be larger than MaxHeapTuplesPerPage
+ * (~290), which is the theoretical maximum for single-page ranges.
+ */
+#define BLOOM_MIN_NDISTINCT_PER_RANGE 16
+
+/*
+ * Used to determine number of distinct items, based on the number of rows
+ * in a page range. The 10% is somewhat similar to what estimate_num_groups
+ * does, so we use the same factor here.
+ */
+#define BLOOM_DEFAULT_NDISTINCT_PER_RANGE -0.1 /* 10% of values */
+
+/*
+ * The 1% value is mostly arbitrary, it just looks nice.
+ */
+#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
+
+#define BloomGetNDistinctPerRange(opts) \
+ ((opts) && (((BloomOptions *) (opts))->nDistinctPerRange != 0) ? \
+ (((BloomOptions *) (opts))->nDistinctPerRange) : \
+ BLOOM_DEFAULT_NDISTINCT_PER_RANGE)
+
+#define BloomGetFalsePositiveRate(opts) \
+ ((opts) && (((BloomOptions *) (opts))->falsePositiveRate != 0.0) ? \
+ (((BloomOptions *) (opts))->falsePositiveRate) : \
+ BLOOM_DEFAULT_FALSE_POSITIVE_RATE)
+
+/*
+ * Bloom Filter
+ *
+ * Represents a bloom filter, built on hashes of the indexed values. That is,
+ * we compute a uint32 hash of the value, and then store this hash into the
+ * bloom filter (and compute additional hashes on it).
+ *
+ * We use an optimisation that initially we store the uint32 values directly,
+ * without the extra hashing step. And only later filling the bitmap space,
+ * we switch to the regular bloom filter mode.
+ *
+ * PHASE_SORTED
+ *
+ * Initially we copy the uint32 hash into the bitmap, regularly sorting the
+ * hash values for fast lookup (we keep at most BLOOM_MAX_UNSORTED unsorted
+ * values).
+ *
+ * The idea is that if we only see very few distinct values, we can store
+ * them in less space compared to the (sparse) bloom filter bitmap. It also
+ * stores them exactly, although that's not a big advantage as almost-empty
+ * bloom filter has false positive rate close to zero anyway.
+ *
+ * PHASE_HASH
+ *
+ * Once we fill the bitmap space in the sorted phase, we switch to the hash
+ * phase, where we actually use the bloom filter. We treat the uint32 hashes
+ * as input values, and hash them again with different seeds (to get the k
+ * hash functions needed for bloom filter).
+ *
+ *
+ * XXX Perhaps we could save a few bytes by using different data types, but
+ * considering the size of the bitmap, the difference is negligible.
+ *
+ * XXX We could also implement "sparse" bloom filters, keeping only the
+ * bytes that are not entirely 0. That might make the "sorted" phase
+ * mostly unnecessary.
+ *
+ * XXX We can also watch the number of bits set in the bloom filter, and
+ * then stop using it (and not store the bitmap, to save space) when the
+ * false positive rate gets too high.
+ */
+typedef struct BloomFilter
+{
+ /* varlena header (do not touch directly!) */
+ int32 vl_len_;
+
+ /* space for various flags (phase, etc.) */
+ uint16 flags;
+
+ /* fields used only in the SORTED phase */
+ uint16 nvalues; /* number of hashes stored (total) */
+ uint16 nsorted; /* number of hashes in the sorted part */
+
+ /* fields for the HASHED phase */
+ uint8 nhashes; /* number of hash functions */
+ uint32 nbits; /* number of bits in the bitmap (size) */
+ uint32 nbits_set; /* number of bits set to 1 */
+
+ /* data of the bloom filter (used both for sorted and hashed phase) */
+ char data[FLEXIBLE_ARRAY_MEMBER];
+
+} BloomFilter;
+
+static BloomFilter *bloom_switch_to_hashing(BloomFilter *filter);
+
+
+/*
+ * bloom_init
+ * Initialize the Bloom Filter, allocate all the memory.
+ *
+ * The filter is initialized with optimal size for ndistinct expected
+ * values, and requested false positive rate. The filter is stored as
+ * varlena.
+ */
+static BloomFilter *
+bloom_init(int ndistinct, double false_positive_rate)
+{
+ Size len;
+ BloomFilter *filter;
+
+ int m; /* number of bits */
+ double k; /* number of hash functions */
+
+ Assert(ndistinct > 0);
+ Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
+
+ m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+
+ /* round m to whole bytes */
+ m = ((m + 7) / 8) * 8;
+
+ /*
+ * round(log(2.0) * m / ndistinct), but assume round() may not be
+ * available on Windows
+ */
+ k = log(2.0) * m / ndistinct;
+ k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
+
+ /*
+ * Allocate the bloom filter with a minimum size 64B (about 40B in the
+ * bitmap part). We require space at least for the header.
+ *
+ * XXX Maybe the 64B min size is not really needed?
+ */
+ len = Max(offsetof(BloomFilter, data), 64);
+
+ filter = (BloomFilter *) palloc0(len);
+
+ filter->flags = 0; /* implies SORTED phase */
+ filter->nhashes = (int) k;
+ filter->nbits = m;
+
+ SET_VARSIZE(filter, len);
+
+ return filter;
+}
+
+/* simple uint32 comparator, for pg_qsort and bsearch */
+static int
+cmp_uint32(const void *a, const void *b)
+{
+ uint32 *ia = (uint32 *) a;
+ uint32 *ib = (uint32 *) b;
+
+ if (*ia == *ib)
+ return 0;
+ else if (*ia < *ib)
+ return -1;
+ else
+ return 1;
+}
+
+/*
+ * bloom_compact
+ * Compact the filter during the 'sorted' phase.
+ *
+ * We sort the uint32 hashes and remove duplicates, for two main reasons.
+ * Firstly, to keep most of the data sorted for bsearch lookups. Secondly,
+ * we try to save space by removing the duplicates, allowing us to stay
+ * in the sorted phase a bit longer.
+ *
+ * We currently don't repalloc the bitmap, i.e. we don't free the memory
+ * here - in the worst case we waste space for up to 32 unsorted hashes
+ * (if all of them are already in the sorted part), so about 128B. We can
+ * either reduce the number of unsorted items (e.g. to 8 hashes, which
+ * would mean 32B), or start doing the repalloc.
+ *
+ * We do however set the varlena length, to minimize the storage needs.
+ */
+static void
+bloom_compact(BloomFilter *filter)
+{
+ int i,
+ nvalues;
+ Size len;
+ uint32 *values;
+
+ /* never call compact on filters in HASH phase */
+ Assert(BLOOM_IS_SORTED(filter));
+
+ /* no chance to compact anything */
+ if (filter->nvalues == filter->nsorted)
+ return;
+
+ values = (uint32 *) filter->data;
+
+ /* TODO optimization: sort only the unsorted part, then merge */
+ pg_qsort(values, filter->nvalues, sizeof(uint32), cmp_uint32);
+
+ nvalues = 1;
+ for (i = 1; i < filter->nvalues; i++)
+ {
+ /* if different from the last value, keep it */
+ if (values[i] != values[nvalues - 1])
+ values[nvalues++] = values[i];
+ }
+
+ filter->nvalues = nvalues;
+ filter->nsorted = nvalues;
+
+ len = offsetof(BloomFilter, data) +
+ (filter->nvalues) * sizeof(uint32);
+
+ SET_VARSIZE(filter, len);
+}
+
+/*
+ * bloom_add_value
+ * Add value to the bloom filter.
+ */
+static BloomFilter *
+bloom_add_value(BloomFilter *filter, uint32 value, bool *updated)
+{
+ int i;
+ uint32 big_h, h, d;
+
+ /* assume 'not updated' by default */
+ Assert(filter);
+
+ /* if we're in the sorted phase, we store the hashes directly */
+ if (BLOOM_IS_SORTED(filter))
+ {
+ /* how many uint32 hashes can we fit into the bitmap */
+ int maxvalues = filter->nbits / (8 * sizeof(uint32));
+
+ /* do not overflow the bitmap space or number of unsorted items */
+ Assert(filter->nvalues <= maxvalues);
+ Assert(filter->nvalues - filter->nsorted <= BLOOM_MAX_UNSORTED);
+
+ /*
+ * In this branch we always update the filter - we either add the
+ * hash to the unsorted part, or switch the filter to hashing.
+ */
+ if (updated)
+ *updated = true;
+
+ /*
+ * If the array is full, or if we reached the limit on unsorted
+ * items, try to compact the filter first, before attempting to
+ * add the new value.
+ */
+ if ((filter->nvalues == maxvalues) ||
+ (filter->nvalues - filter->nsorted == BLOOM_MAX_UNSORTED))
+ bloom_compact(filter);
+
+ /*
+ * Can we squeeze one more uint32 hash into the bitmap? Also make
+ * sure there's enough space in the bytea value first.
+ */
+ if (filter->nvalues < maxvalues)
+ {
+ Size len = VARSIZE_ANY(filter);
+ Size need = offsetof(BloomFilter, data) +
+ (filter->nvalues + 1) * sizeof(uint32);
+
+ /*
+ * We don't double the size here, as in the first place we care about
+ * reducing storage requirements, and the doubling happens automatically
+ * in memory contexts (so the repalloc should be cheap in most cases).
+ */
+ if (len < need)
+ {
+ filter = (BloomFilter *) repalloc(filter, need);
+ SET_VARSIZE(filter, need);
+ }
+
+ /* copy the new value into the filter */
+ memcpy(&filter->data[filter->nvalues * sizeof(uint32)],
+ &value, sizeof(uint32));
+
+ filter->nvalues++;
+
+ /* we're done */
+ return filter;
+ }
+
+ /* can't add any more exact hashes, so switch to hashing */
+ filter = bloom_switch_to_hashing(filter);
+ }
+
+ /* we better be in the hashing phase */
+ Assert(BLOOM_IS_HASHED(filter));
+
+ /* compute the hashes, used for the bloom filter */
+ big_h = ((uint32) DatumGetInt64(hash_uint32(value)));
+
+ h = big_h % filter->nbits;
+ d = big_h % (filter->nbits - 1);
+
+ /* compute the requested number of hashes */
+ for (i = 0; i < filter->nhashes; i++)
+ {
+ int byte = (h / 8);
+ int bit = (h % 8);
+
+ /* if the bit is not set, set it and remember we did that */
+ if (! (filter->data[byte] & (0x01 << bit)))
+ {
+ filter->data[byte] |= (0x01 << bit);
+ filter->nbits_set++;
+ if (updated)
+ *updated = true;
+ }
+
+ /* next bit */
+ h += d++;
+ if (h >= filter->nbits)
+ h -= filter->nbits;
+
+ if (d == filter->nbits)
+ d = 0;
+ }
+
+ return filter;
+}
+
+/*
+ * bloom_switch_to_hashing
+ * Switch the bloom filter from sorted to hashing mode.
+ */
+static BloomFilter *
+bloom_switch_to_hashing(BloomFilter *filter)
+{
+ int i;
+ uint32 *values;
+ Size len;
+ BloomFilter *newfilter;
+
+ Assert(filter->nbits % 8 == 0);
+
+ /*
+ * The new filter is allocated with all the memory, directly into
+ * the HASH phase.
+ */
+ len = offsetof(BloomFilter, data) + (filter->nbits / 8);
+
+ newfilter = (BloomFilter *) palloc0(len);
+
+ newfilter->nhashes = filter->nhashes;
+ newfilter->nbits = filter->nbits;
+ newfilter->flags |= BLOOM_FLAG_PHASE_HASH;
+
+ SET_VARSIZE(newfilter, len);
+
+ values = (uint32 *) filter->data;
+
+ for (i = 0; i < filter->nvalues; i++)
+ /* ignore the return value here, re don't repalloc in hashing mode */
+ bloom_add_value(newfilter, values[i], NULL);
+
+ /* free the original filter, return the newly allocated one */
+ pfree(filter);
+
+ return newfilter;
+}
+
+/*
+ * bloom_contains_value
+ * Check if the bloom filter contains a particular value.
+ */
+static bool
+bloom_contains_value(BloomFilter *filter, uint32 value)
+{
+ int i;
+ uint32 big_h, h, d;
+
+ Assert(filter);
+
+ /* in sorted mode we simply search the two arrays (sorted, unsorted) */
+ if (BLOOM_IS_SORTED(filter))
+ {
+ int i;
+ uint32 *values = (uint32 *) filter->data;
+
+ /* first search through the sorted part */
+ if ((filter->nsorted > 0) &&
+ (bsearch(&value, values, filter->nsorted, sizeof(uint32), cmp_uint32) != NULL))
+ return true;
+
+ /* now search through the unsorted part - linear search */
+ for (i = filter->nsorted; i < filter->nvalues; i++)
+ {
+ if (value == values[i])
+ return true;
+ }
+
+ /* nothing found */
+ return false;
+ }
+
+ /* now the regular hashing mode */
+ Assert(BLOOM_IS_HASHED(filter));
+
+ big_h = ((uint32) DatumGetInt64(hash_uint32(value)));
+
+ h = big_h % filter->nbits;
+ d = big_h % (filter->nbits - 1);
+
+ /* compute the requested number of hashes */
+ for (i = 0; i < filter->nhashes; i++)
+ {
+ int byte = (h / 8);
+ int bit = (h % 8);
+
+ /* if the bit is not set, the value is not there */
+ if (! (filter->data[byte] & (0x01 << bit)))
+ return false;
+
+ /* next bit */
+ h += d++;
+ if (h >= filter->nbits)
+ h -= filter->nbits;
+
+ if (d == filter->nbits)
+ d = 0;
+ }
+
+ /* all hashes found in bloom filter */
+ return true;
+}
+
+typedef struct BloomOpaque
+{
+ /*
+ * XXX At this point we only need a single proc (to compute the hash),
+ * but let's keep the array just like inclusion and minman opclasses,
+ * for consistency. We may need additional procs in the future.
+ */
+ FmgrInfo extra_procinfos[BLOOM_MAX_PROCNUMS];
+ bool extra_proc_missing[BLOOM_MAX_PROCNUMS];
+} BloomOpaque;
+
+static FmgrInfo *bloom_get_procinfo(BrinDesc *bdesc, uint16 attno,
+ uint16 procnum);
+
+
+Datum
+brin_bloom_opcinfo(PG_FUNCTION_ARGS)
+{
+ BrinOpcInfo *result;
+
+ /*
+ * opaque->strategy_procinfos is initialized lazily; here it is set to
+ * all-uninitialized by palloc0 which sets fn_oid to InvalidOid.
+ *
+ * bloom indexes only store the filter as a single BYTEA column
+ */
+
+ result = palloc0(MAXALIGN(SizeofBrinOpcInfo(1)) +
+ sizeof(BloomOpaque));
+ result->oi_nstored = 1;
+ result->oi_regular_nulls = true;
+ result->oi_opaque = (BloomOpaque *)
+ MAXALIGN((char *) result + SizeofBrinOpcInfo(1));
+ result->oi_typcache[0] = lookup_type_cache(BYTEAOID, 0);
+
+ PG_RETURN_POINTER(result);
+}
+
+/*
+ * brin_bloom_get_ndistinct
+ * Determine the ndistinct value used to size bloom filter.
+ *
+ * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * if it's negative, it's assumed to be relative to maximum number of
+ * tuples in the range (assuming each page gets MaxHeapTuplesPerPage
+ * tuples, which is likely a significant over-estimate). We also clamp
+ * the value, not to over-size the bloom filter unnecessarily.
+ *
+ * XXX We can only do this when the pagesPerRange value was supplied.
+ * If it wasn't, it has to be a read-only access to the index, in which
+ * case we don't really care. But perhaps we should fall-back to the
+ * default pagesPerRange value?
+ *
+ * XXX We might also fetch info about ndistinct estimate for the column,
+ * and compute the expected number of distinct values in a range. But
+ * that may be tricky due to data being sorted in various ways, so it
+ * seems better to rely on the upper estimate.
+ */
+static int
+brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
+{
+ double ndistinct;
+ double maxtuples;
+ BlockNumber pagesPerRange;
+
+ pagesPerRange = BrinGetPagesPerRange(bdesc->bd_index);
+ ndistinct = BloomGetNDistinctPerRange(opts);
+
+ Assert(BlockNumberIsValid(pagesPerRange));
+
+ maxtuples = MaxHeapTuplesPerPage * pagesPerRange;
+
+ /*
+ * Similarly to n_distinct, negative values are relative - in this
+ * case to maximum number of tuples in the page range (maxtuples).
+ */
+ if (ndistinct < 0)
+ ndistinct = (-ndistinct) * maxtuples;
+
+ /*
+ * Positive values are to be used directly, but we still apply a
+ * couple of safeties no to use unreasonably small bloom filters.
+ */
+ ndistinct = Max(ndistinct, BLOOM_MIN_NDISTINCT_PER_RANGE);
+
+ /*
+ * And don't use more than the maximum possible number of tuples,
+ * in the range, which would be entirely wasteful.
+ */
+ ndistinct = Min(ndistinct, maxtuples);
+
+ return (int) ndistinct;
+}
+
+static double
+brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
+{
+ return BloomGetFalsePositiveRate(opts);
+}
+
+/*
+ * Examine the given index tuple (which contains partial status of a certain
+ * page range) by comparing it to the given value that comes from another heap
+ * tuple. If the new value is outside the bloom filter specified by the
+ * existing tuple values, update the index tuple and return true. Otherwise,
+ * return false and do not modify in this case.
+ */
+Datum
+brin_bloom_add_value(PG_FUNCTION_ARGS)
+{
+ BrinDesc *bdesc = (BrinDesc *) PG_GETARG_POINTER(0);
+ BrinValues *column = (BrinValues *) PG_GETARG_POINTER(1);
+ Datum newval = PG_GETARG_DATUM(2);
+ bool isnull PG_USED_FOR_ASSERTS_ONLY = PG_GETARG_DATUM(3);
+ BloomOptions *opts = (BloomOptions *) PG_GET_OPCLASS_OPTIONS();
+ Oid colloid = PG_GET_COLLATION();
+ FmgrInfo *hashFn;
+ uint32 hashValue;
+ bool updated = false;
+ AttrNumber attno;
+ BloomFilter *filter;
+
+ Assert(!isnull);
+
+ attno = column->bv_attno;
+
+ /*
+ * If this is the first non-null value, we need to initialize the bloom
+ * filter. Otherwise just extract the existing bloom filter from BrinValues.
+ */
+ if (column->bv_allnulls)
+ {
+ filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
+ brin_bloom_get_fp_rate(bdesc, opts));
+ column->bv_values[0] = PointerGetDatum(filter);
+ column->bv_allnulls = false;
+ updated = true;
+ }
+ else
+ filter = (BloomFilter *) PG_DETOAST_DATUM(column->bv_values[0]);
+
+ /*
+ * Compute the hash of the new value, using the supplied hash function,
+ * and then add the hash value to the bloom filter.
+ */
+ hashFn = bloom_get_procinfo(bdesc, attno, PROCNUM_HASH);
+
+ hashValue = DatumGetUInt32(FunctionCall1Coll(hashFn, colloid, newval));
+
+ filter = bloom_add_value(filter, hashValue, &updated);
+
+ column->bv_values[0] = PointerGetDatum(filter);
+
+ PG_RETURN_BOOL(updated);
+}
+
+/*
+ * Given an index tuple corresponding to a certain page range and a scan key,
+ * return whether the scan key is consistent with the index tuple's bloom
+ * filter. Return true if so, false otherwise.
+ */
+Datum
+brin_bloom_consistent(PG_FUNCTION_ARGS)
+{
+ BrinDesc *bdesc = (BrinDesc *) PG_GETARG_POINTER(0);
+ BrinValues *column = (BrinValues *) PG_GETARG_POINTER(1);
+ ScanKey *keys = (ScanKey *) PG_GETARG_POINTER(2);
+ int nkeys = PG_GETARG_INT32(3);
+ Oid colloid = PG_GET_COLLATION();
+ AttrNumber attno;
+ Datum value;
+ Datum matches;
+ FmgrInfo *finfo;
+ uint32 hashValue;
+ BloomFilter *filter;
+ int keyno;
+
+ filter = (BloomFilter *) PG_DETOAST_DATUM(column->bv_values[0]);
+
+ Assert(filter);
+
+ matches = true;
+
+ for (keyno = 0; keyno < nkeys; keyno++)
+ {
+ ScanKey key = keys[keyno];
+
+ /* NULL keys are handled and filtered-out in bringetbitmap */
+ Assert(!(key->sk_flags & SK_ISNULL));
+
+ attno = key->sk_attno;
+ value = key->sk_argument;
+
+ switch (key->sk_strategy)
+ {
+ case BloomEqualStrategyNumber:
+
+ /*
+ * In the equality case (WHERE col = someval), we want to return
+ * the current page range if the minimum value in the range <=
+ * scan key, and the maximum value >= scan key.
+ */
+ finfo = bloom_get_procinfo(bdesc, attno, PROCNUM_HASH);
+
+ hashValue = DatumGetUInt32(FunctionCall1Coll(finfo, colloid, value));
+ matches &= bloom_contains_value(filter, hashValue);
+
+ break;
+ default:
+ /* shouldn't happen */
+ elog(ERROR, "invalid strategy number %d", key->sk_strategy);
+ matches = 0;
+ break;
+ }
+
+ if (!matches)
+ break;
+ }
+
+ PG_RETURN_DATUM(matches);
+}
+
+/*
+ * Given two BrinValues, update the first of them as a union of the summary
+ * values contained in both. The second one is untouched.
+ *
+ * XXX We assume the bloom filters have the same parameters fow now. In the
+ * future we should have 'can union' function, to decide if we can combine
+ * two particular bloom filters.
+ */
+Datum
+brin_bloom_union(PG_FUNCTION_ARGS)
+{
+ BrinValues *col_a = (BrinValues *) PG_GETARG_POINTER(1);
+ BrinValues *col_b = (BrinValues *) PG_GETARG_POINTER(2);
+ BloomFilter *filter_a;
+ BloomFilter *filter_b;
+
+ Assert(col_a->bv_attno == col_b->bv_attno);
+ Assert(!col_a->bv_allnulls && !col_b->bv_allnulls);
+
+ filter_a = (BloomFilter *) PG_DETOAST_DATUM(col_a->bv_values[0]);
+ filter_b = (BloomFilter *) PG_DETOAST_DATUM(col_b->bv_values[0]);
+
+ /* make sure neither of the bloom filters is NULL */
+ Assert(filter_a && filter_b);
+ Assert(filter_a->nbits == filter_b->nbits);
+
+ /*
+ * Merging of the filters depends on the phase of both filters.
+ */
+ if (BLOOM_IS_SORTED(filter_b))
+ {
+ /*
+ * Simply read all items from 'b' and add them to 'a' (the phase of
+ * 'a' does not really matter).
+ */
+ int i;
+ uint32 *values = (uint32 *) filter_b->data;
+
+ for (i = 0; i < filter_b->nvalues; i++)
+ filter_a = bloom_add_value(filter_a, values[i], NULL);
+
+ col_a->bv_values[0] = PointerGetDatum(filter_a);
+ }
+ else if (BLOOM_IS_SORTED(filter_a))
+ {
+ /*
+ * 'b' hashed, 'a' sorted - copy 'b' into 'a' and then add all values
+ * from 'a' into the new copy.
+ */
+ int i;
+ BloomFilter *filter_c;
+ uint32 *values = (uint32 *) filter_a->data;
+
+ filter_c = (BloomFilter *) PG_DETOAST_DATUM(datumCopy(PointerGetDatum(filter_b), false, -1));
+
+ for (i = 0; i < filter_a->nvalues; i++)
+ filter_c = bloom_add_value(filter_c, values[i], NULL);
+
+ col_a->bv_values[0] = PointerGetDatum(filter_c);
+ }
+ else if (BLOOM_IS_HASHED(filter_a))
+ {
+ /*
+ * 'b' hashed, 'a' hashed - merge the bitmaps by OR
+ */
+ int i;
+ int nbytes = (filter_a->nbits + 7) / 8;
+
+ /* we better have "compatible" bloom filters */
+ Assert(filter_a->nbits == filter_b->nbits);
+ Assert(filter_a->nhashes == filter_b->nhashes);
+
+ for (i = 0; i < nbytes; i++)
+ filter_a->data[i] |= filter_b->data[i];
+ }
+
+ PG_RETURN_VOID();
+}
+
+/*
+ * Cache and return inclusion opclass support procedure
+ *
+ * Return the procedure corresponding to the given function support number
+ * or null if it is not exists.
+ */
+static FmgrInfo *
+bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
+{
+ BloomOpaque *opaque;
+ uint16 basenum = procnum - PROCNUM_BASE;
+
+ /*
+ * We cache these in the opaque struct, to avoid repetitive syscache
+ * lookups.
+ */
+ opaque = (BloomOpaque *) bdesc->bd_info[attno - 1]->oi_opaque;
+
+ /*
+ * If we already searched for this proc and didn't find it, don't bother
+ * searching again.
+ */
+ if (opaque->extra_proc_missing[basenum])
+ return NULL;
+
+ if (opaque->extra_procinfos[basenum].fn_oid == InvalidOid)
+ {
+ if (RegProcedureIsValid(index_getprocid(bdesc->bd_index, attno,
+ procnum)))
+ {
+ fmgr_info_copy(&opaque->extra_procinfos[basenum],
+ index_getprocinfo(bdesc->bd_index, attno, procnum),
+ bdesc->bd_context);
+ }
+ else
+ {
+ opaque->extra_proc_missing[basenum] = true;
+ return NULL;
+ }
+ }
+
+ return &opaque->extra_procinfos[basenum];
+}
+
+Datum
+brin_bloom_options(PG_FUNCTION_ARGS)
+{
+ local_relopts *relopts = (local_relopts *) PG_GETARG_POINTER(0);
+
+ init_local_reloptions(relopts, sizeof(BloomOptions));
+
+ add_local_real_reloption(relopts, "n_distinct_per_range",
+ "number of distinct items expected in a BRIN page range",
+ BLOOM_DEFAULT_NDISTINCT_PER_RANGE,
+ -1.0, INT_MAX, offsetof(BloomOptions, nDistinctPerRange));
+
+ add_local_real_reloption(relopts, "false_positive_rate",
+ "desired false-positive rate for the bloom filters",
+ BLOOM_DEFAULT_FALSE_POSITIVE_RATE,
+ 0.001, 1.0, offsetof(BloomOptions, falsePositiveRate));
+
+ PG_RETURN_VOID();
+}
diff --git a/src/include/access/brin.h b/src/include/access/brin.h
index 0987878dd2..b02298c0aa 100644
--- a/src/include/access/brin.h
+++ b/src/include/access/brin.h
@@ -22,6 +22,8 @@ typedef struct BrinOptions
int32 vl_len_; /* varlena header (do not touch directly!) */
BlockNumber pagesPerRange;
bool autosummarize;
+ double nDistinctPerRange; /* number of distinct values per range */
+ double falsePositiveRate; /* false positive for bloom filter */
} BrinOptions;
diff --git a/src/include/access/brin_internal.h b/src/include/access/brin_internal.h
index 7c4f3da0a0..e3c9d47503 100644
--- a/src/include/access/brin_internal.h
+++ b/src/include/access/brin_internal.h
@@ -58,6 +58,10 @@ typedef struct BrinDesc
/* total number of Datum entries that are stored on-disk for all columns */
int bd_totalstored;
+ /* parameters for sizing bloom filter (BRIN bloom opclasses) */
+ double bd_nDistinctPerRange;
+ double bd_falsePositiveRange;
+
/* per-column info; bd_tupdesc->natts entries long */
BrinOpcInfo *bd_info[FLEXIBLE_ARRAY_MEMBER];
} BrinDesc;
diff --git a/src/include/catalog/pg_amop.dat b/src/include/catalog/pg_amop.dat
index 1dfb6fd373..af6891e348 100644
--- a/src/include/catalog/pg_amop.dat
+++ b/src/include/catalog/pg_amop.dat
@@ -1705,6 +1705,11 @@
amoprighttype => 'bytea', amopstrategy => '5', amopopr => '>(bytea,bytea)',
amopmethod => 'brin' },
+# bloom bytea
+{ amopfamily => 'brin/bytea_bloom_ops', amoplefttype => 'bytea',
+ amoprighttype => 'bytea', amopstrategy => '1', amopopr => '=(bytea,bytea)',
+ amopmethod => 'brin' },
+
# minmax "char"
{ amopfamily => 'brin/char_minmax_ops', amoplefttype => 'char',
amoprighttype => 'char', amopstrategy => '1', amopopr => '<(char,char)',
@@ -1722,6 +1727,11 @@
amoprighttype => 'char', amopstrategy => '5', amopopr => '>(char,char)',
amopmethod => 'brin' },
+# bloom "char"
+{ amopfamily => 'brin/char_bloom_ops', amoplefttype => 'char',
+ amoprighttype => 'char', amopstrategy => '1', amopopr => '=(char,char)',
+ amopmethod => 'brin' },
+
# minmax name
{ amopfamily => 'brin/name_minmax_ops', amoplefttype => 'name',
amoprighttype => 'name', amopstrategy => '1', amopopr => '<(name,name)',
@@ -1739,6 +1749,11 @@
amoprighttype => 'name', amopstrategy => '5', amopopr => '>(name,name)',
amopmethod => 'brin' },
+# bloom name
+{ amopfamily => 'brin/name_bloom_ops', amoplefttype => 'name',
+ amoprighttype => 'name', amopstrategy => '1', amopopr => '=(name,name)',
+ amopmethod => 'brin' },
+
# minmax integer
{ amopfamily => 'brin/integer_minmax_ops', amoplefttype => 'int8',
@@ -1885,6 +1900,44 @@
amoprighttype => 'int8', amopstrategy => '5', amopopr => '>(int4,int8)',
amopmethod => 'brin' },
+# bloom integer
+
+{ amopfamily => 'brin/integer_bloom_ops', amoplefttype => 'int8',
+ amoprighttype => 'int8', amopstrategy => '1', amopopr => '=(int8,int8)',
+ amopmethod => 'brin' },
+
+{ amopfamily => 'brin/integer_bloom_ops', amoplefttype => 'int8',
+ amoprighttype => 'int2', amopstrategy => '1', amopopr => '=(int8,int2)',
+ amopmethod => 'brin' },
+
+{ amopfamily => 'brin/integer_bloom_ops', amoplefttype => 'int8',
+ amoprighttype => 'int4', amopstrategy => '1', amopopr => '=(int8,int4)',
+ amopmethod => 'brin' },
+
+{ amopfamily => 'brin/integer_bloom_ops', amoplefttype => 'int2',
+ amoprighttype => 'int2', amopstrategy => '1', amopopr => '=(int2,int2)',
+ amopmethod => 'brin' },
+
+{ amopfamily => 'brin/integer_bloom_ops', amoplefttype => 'int2',
+ amoprighttype => 'int8', amopstrategy => '1', amopopr => '=(int2,int8)',
+ amopmethod => 'brin' },
+
+{ amopfamily => 'brin/integer_bloom_ops', amoplefttype => 'int2',
+ amoprighttype => 'int4', amopstrategy => '1', amopopr => '=(int2,int4)',
+ amopmethod => 'brin' },
+
+{ amopfamily => 'brin/integer_bloom_ops', amoplefttype => 'int4',
+ amoprighttype => 'int4', amopstrategy => '1', amopopr => '=(int4,int4)',
+ amopmethod => 'brin' },
+
+{ amopfamily => 'brin/integer_bloom_ops', amoplefttype => 'int4',
+ amoprighttype => 'int2', amopstrategy => '1', amopopr => '=(int4,int2)',
+ amopmethod => 'brin' },
+
+{ amopfamily => 'brin/integer_bloom_ops', amoplefttype => 'int4',
+ amoprighttype => 'int8', amopstrategy => '1', amopopr => '=(int4,int8)',
+ amopmethod => 'brin' },
+
# minmax text
{ amopfamily => 'brin/text_minmax_ops', amoplefttype => 'text',
amoprighttype => 'text', amopstrategy => '1', amopopr => '<(text,text)',
@@ -1902,6 +1955,11 @@
amoprighttype => 'text', amopstrategy => '5', amopopr => '>(text,text)',
amopmethod => 'brin' },
+# bloom text
+{ amopfamily => 'brin/text_bloom_ops', amoplefttype => 'text',
+ amoprighttype => 'text', amopstrategy => '1', amopopr => '=(text,text)',
+ amopmethod => 'brin' },
+
# minmax oid
{ amopfamily => 'brin/oid_minmax_ops', amoplefttype => 'oid',
amoprighttype => 'oid', amopstrategy => '1', amopopr => '<(oid,oid)',
@@ -1919,6 +1977,11 @@
amoprighttype => 'oid', amopstrategy => '5', amopopr => '>(oid,oid)',
amopmethod => 'brin' },
+# bloom oid
+{ amopfamily => 'brin/oid_bloom_ops', amoplefttype => 'oid',
+ amoprighttype => 'oid', amopstrategy => '1', amopopr => '=(oid,oid)',
+ amopmethod => 'brin' },
+
# minmax tid
{ amopfamily => 'brin/tid_minmax_ops', amoplefttype => 'tid',
amoprighttype => 'tid', amopstrategy => '1', amopopr => '<(tid,tid)',
@@ -2002,6 +2065,20 @@
amoprighttype => 'float8', amopstrategy => '5', amopopr => '>(float8,float8)',
amopmethod => 'brin' },
+# bloom float
+{ amopfamily => 'brin/float_bloom_ops', amoplefttype => 'float4',
+ amoprighttype => 'float4', amopstrategy => '1', amopopr => '=(float4,float4)',
+ amopmethod => 'brin' },
+{ amopfamily => 'brin/float_bloom_ops', amoplefttype => 'float4',
+ amoprighttype => 'float8', amopstrategy => '1', amopopr => '=(float4,float8)',
+ amopmethod => 'brin' },
+{ amopfamily => 'brin/float_bloom_ops', amoplefttype => 'float8',
+ amoprighttype => 'float4', amopstrategy => '1', amopopr => '=(float8,float4)',
+ amopmethod => 'brin' },
+{ amopfamily => 'brin/float_bloom_ops', amoplefttype => 'float8',
+ amoprighttype => 'float8', amopstrategy => '1', amopopr => '=(float8,float8)',
+ amopmethod => 'brin' },
+
# minmax macaddr
{ amopfamily => 'brin/macaddr_minmax_ops', amoplefttype => 'macaddr',
amoprighttype => 'macaddr', amopstrategy => '1',
@@ -2019,6 +2096,11 @@
amoprighttype => 'macaddr', amopstrategy => '5',
amopopr => '>(macaddr,macaddr)', amopmethod => 'brin' },
+# bloom macaddr
+{ amopfamily => 'brin/macaddr_bloom_ops', amoplefttype => 'macaddr',
+ amoprighttype => 'macaddr', amopstrategy => '1',
+ amopopr => '=(macaddr,macaddr)', amopmethod => 'brin' },
+
# minmax macaddr8
{ amopfamily => 'brin/macaddr8_minmax_ops', amoplefttype => 'macaddr8',
amoprighttype => 'macaddr8', amopstrategy => '1',
@@ -2036,6 +2118,11 @@
amoprighttype => 'macaddr8', amopstrategy => '5',
amopopr => '>(macaddr8,macaddr8)', amopmethod => 'brin' },
+# bloom macaddr8
+{ amopfamily => 'brin/macaddr8_bloom_ops', amoplefttype => 'macaddr8',
+ amoprighttype => 'macaddr8', amopstrategy => '1',
+ amopopr => '=(macaddr8,macaddr8)', amopmethod => 'brin' },
+
# minmax inet
{ amopfamily => 'brin/network_minmax_ops', amoplefttype => 'inet',
amoprighttype => 'inet', amopstrategy => '1', amopopr => '<(inet,inet)',
@@ -2053,6 +2140,11 @@
amoprighttype => 'inet', amopstrategy => '5', amopopr => '>(inet,inet)',
amopmethod => 'brin' },
+# bloom inet
+{ amopfamily => 'brin/network_bloom_ops', amoplefttype => 'inet',
+ amoprighttype => 'inet', amopstrategy => '1', amopopr => '=(inet,inet)',
+ amopmethod => 'brin' },
+
# inclusion inet
{ amopfamily => 'brin/network_inclusion_ops', amoplefttype => 'inet',
amoprighttype => 'inet', amopstrategy => '3', amopopr => '&&(inet,inet)',
@@ -2090,6 +2182,11 @@
amoprighttype => 'bpchar', amopstrategy => '5', amopopr => '>(bpchar,bpchar)',
amopmethod => 'brin' },
+# bloom character
+{ amopfamily => 'brin/bpchar_bloom_ops', amoplefttype => 'bpchar',
+ amoprighttype => 'bpchar', amopstrategy => '1', amopopr => '=(bpchar,bpchar)',
+ amopmethod => 'brin' },
+
# minmax time without time zone
{ amopfamily => 'brin/time_minmax_ops', amoplefttype => 'time',
amoprighttype => 'time', amopstrategy => '1', amopopr => '<(time,time)',
@@ -2107,6 +2204,11 @@
amoprighttype => 'time', amopstrategy => '5', amopopr => '>(time,time)',
amopmethod => 'brin' },
+# bloom time without time zone
+{ amopfamily => 'brin/time_bloom_ops', amoplefttype => 'time',
+ amoprighttype => 'time', amopstrategy => '1', amopopr => '=(time,time)',
+ amopmethod => 'brin' },
+
# minmax datetime (date, timestamp, timestamptz)
{ amopfamily => 'brin/datetime_minmax_ops', amoplefttype => 'timestamp',
@@ -2253,6 +2355,44 @@
amoprighttype => 'timestamptz', amopstrategy => '5',
amopopr => '>(timestamptz,timestamptz)', amopmethod => 'brin' },
+# bloom datetime (date, timestamp, timestamptz)
+
+{ amopfamily => 'brin/datetime_bloom_ops', amoplefttype => 'timestamp',
+ amoprighttype => 'timestamp', amopstrategy => '1',
+ amopopr => '=(timestamp,timestamp)', amopmethod => 'brin' },
+
+{ amopfamily => 'brin/datetime_bloom_ops', amoplefttype => 'timestamp',
+ amoprighttype => 'date', amopstrategy => '1', amopopr => '=(timestamp,date)',
+ amopmethod => 'brin' },
+
+{ amopfamily => 'brin/datetime_bloom_ops', amoplefttype => 'timestamp',
+ amoprighttype => 'timestamptz', amopstrategy => '1',
+ amopopr => '=(timestamp,timestamptz)', amopmethod => 'brin' },
+
+{ amopfamily => 'brin/datetime_bloom_ops', amoplefttype => 'date',
+ amoprighttype => 'date', amopstrategy => '1', amopopr => '=(date,date)',
+ amopmethod => 'brin' },
+
+{ amopfamily => 'brin/datetime_bloom_ops', amoplefttype => 'date',
+ amoprighttype => 'timestamp', amopstrategy => '1',
+ amopopr => '=(date,timestamp)', amopmethod => 'brin' },
+
+{ amopfamily => 'brin/datetime_bloom_ops', amoplefttype => 'date',
+ amoprighttype => 'timestamptz', amopstrategy => '1',
+ amopopr => '=(date,timestamptz)', amopmethod => 'brin' },
+
+{ amopfamily => 'brin/datetime_bloom_ops', amoplefttype => 'timestamptz',
+ amoprighttype => 'date', amopstrategy => '1',
+ amopopr => '=(timestamptz,date)', amopmethod => 'brin' },
+
+{ amopfamily => 'brin/datetime_bloom_ops', amoplefttype => 'timestamptz',
+ amoprighttype => 'timestamp', amopstrategy => '1',
+ amopopr => '=(timestamptz,timestamp)', amopmethod => 'brin' },
+
+{ amopfamily => 'brin/datetime_bloom_ops', amoplefttype => 'timestamptz',
+ amoprighttype => 'timestamptz', amopstrategy => '1',
+ amopopr => '=(timestamptz,timestamptz)', amopmethod => 'brin' },
+
# minmax interval
{ amopfamily => 'brin/interval_minmax_ops', amoplefttype => 'interval',
amoprighttype => 'interval', amopstrategy => '1',
@@ -2270,6 +2410,11 @@
amoprighttype => 'interval', amopstrategy => '5',
amopopr => '>(interval,interval)', amopmethod => 'brin' },
+# bloom interval
+{ amopfamily => 'brin/interval_bloom_ops', amoplefttype => 'interval',
+ amoprighttype => 'interval', amopstrategy => '1',
+ amopopr => '=(interval,interval)', amopmethod => 'brin' },
+
# minmax time with time zone
{ amopfamily => 'brin/timetz_minmax_ops', amoplefttype => 'timetz',
amoprighttype => 'timetz', amopstrategy => '1', amopopr => '<(timetz,timetz)',
@@ -2287,6 +2432,11 @@
amoprighttype => 'timetz', amopstrategy => '5', amopopr => '>(timetz,timetz)',
amopmethod => 'brin' },
+# bloom time with time zone
+{ amopfamily => 'brin/timetz_bloom_ops', amoplefttype => 'timetz',
+ amoprighttype => 'timetz', amopstrategy => '1', amopopr => '=(timetz,timetz)',
+ amopmethod => 'brin' },
+
# minmax bit
{ amopfamily => 'brin/bit_minmax_ops', amoplefttype => 'bit',
amoprighttype => 'bit', amopstrategy => '1', amopopr => '<(bit,bit)',
@@ -2338,6 +2488,11 @@
amoprighttype => 'numeric', amopstrategy => '5',
amopopr => '>(numeric,numeric)', amopmethod => 'brin' },
+# bloom numeric
+{ amopfamily => 'brin/numeric_bloom_ops', amoplefttype => 'numeric',
+ amoprighttype => 'numeric', amopstrategy => '1',
+ amopopr => '=(numeric,numeric)', amopmethod => 'brin' },
+
# minmax uuid
{ amopfamily => 'brin/uuid_minmax_ops', amoplefttype => 'uuid',
amoprighttype => 'uuid', amopstrategy => '1', amopopr => '<(uuid,uuid)',
@@ -2355,6 +2510,11 @@
amoprighttype => 'uuid', amopstrategy => '5', amopopr => '>(uuid,uuid)',
amopmethod => 'brin' },
+# bloom uuid
+{ amopfamily => 'brin/uuid_bloom_ops', amoplefttype => 'uuid',
+ amoprighttype => 'uuid', amopstrategy => '1', amopopr => '=(uuid,uuid)',
+ amopmethod => 'brin' },
+
# inclusion range types
{ amopfamily => 'brin/range_inclusion_ops', amoplefttype => 'anyrange',
amoprighttype => 'anyrange', amopstrategy => '1',
@@ -2416,6 +2576,11 @@
amoprighttype => 'pg_lsn', amopstrategy => '5', amopopr => '>(pg_lsn,pg_lsn)',
amopmethod => 'brin' },
+# bloom pg_lsn
+{ amopfamily => 'brin/pg_lsn_bloom_ops', amoplefttype => 'pg_lsn',
+ amoprighttype => 'pg_lsn', amopstrategy => '1', amopopr => '=(pg_lsn,pg_lsn)',
+ amopmethod => 'brin' },
+
# inclusion box
{ amopfamily => 'brin/box_inclusion_ops', amoplefttype => 'box',
amoprighttype => 'box', amopstrategy => '1', amopopr => '<<(box,box)',
diff --git a/src/include/catalog/pg_amproc.dat b/src/include/catalog/pg_amproc.dat
index 37b580883f..c24a80545a 100644
--- a/src/include/catalog/pg_amproc.dat
+++ b/src/include/catalog/pg_amproc.dat
@@ -770,6 +770,24 @@
{ amprocfamily => 'brin/bytea_minmax_ops', amproclefttype => 'bytea',
amprocrighttype => 'bytea', amprocnum => '4', amproc => 'brin_minmax_union' },
+# bloom bytea
+{ amprocfamily => 'brin/bytea_bloom_ops', amproclefttype => 'bytea',
+ amprocrighttype => 'bytea', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/bytea_bloom_ops', amproclefttype => 'bytea',
+ amprocrighttype => 'bytea', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/bytea_bloom_ops', amproclefttype => 'bytea',
+ amprocrighttype => 'bytea', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/bytea_bloom_ops', amproclefttype => 'bytea',
+ amprocrighttype => 'bytea', amprocnum => '4', amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/bytea_bloom_ops', amproclefttype => 'bytea',
+ amprocrighttype => 'bytea', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/bytea_bloom_ops', amproclefttype => 'bytea',
+ amprocrighttype => 'bytea', amprocnum => '11', amproc => 'hashvarlena' },
+
# minmax "char"
{ amprocfamily => 'brin/char_minmax_ops', amproclefttype => 'char',
amprocrighttype => 'char', amprocnum => '1',
@@ -783,6 +801,24 @@
{ amprocfamily => 'brin/char_minmax_ops', amproclefttype => 'char',
amprocrighttype => 'char', amprocnum => '4', amproc => 'brin_minmax_union' },
+# bloom "char"
+{ amprocfamily => 'brin/char_bloom_ops', amproclefttype => 'char',
+ amprocrighttype => 'char', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/char_bloom_ops', amproclefttype => 'char',
+ amprocrighttype => 'char', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/char_bloom_ops', amproclefttype => 'char',
+ amprocrighttype => 'char', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/char_bloom_ops', amproclefttype => 'char',
+ amprocrighttype => 'char', amprocnum => '4', amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/char_bloom_ops', amproclefttype => 'char',
+ amprocrighttype => 'char', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/char_bloom_ops', amproclefttype => 'char',
+ amprocrighttype => 'char', amprocnum => '11', amproc => 'hashchar' },
+
# minmax name
{ amprocfamily => 'brin/name_minmax_ops', amproclefttype => 'name',
amprocrighttype => 'name', amprocnum => '1',
@@ -796,6 +832,24 @@
{ amprocfamily => 'brin/name_minmax_ops', amproclefttype => 'name',
amprocrighttype => 'name', amprocnum => '4', amproc => 'brin_minmax_union' },
+# bloom name
+{ amprocfamily => 'brin/name_bloom_ops', amproclefttype => 'name',
+ amprocrighttype => 'name', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/name_bloom_ops', amproclefttype => 'name',
+ amprocrighttype => 'name', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/name_bloom_ops', amproclefttype => 'name',
+ amprocrighttype => 'name', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/name_bloom_ops', amproclefttype => 'name',
+ amprocrighttype => 'name', amprocnum => '4', amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/name_bloom_ops', amproclefttype => 'name',
+ amprocrighttype => 'name', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/name_bloom_ops', amproclefttype => 'name',
+ amprocrighttype => 'name', amprocnum => '11', amproc => 'hashname' },
+
# minmax integer: int2, int4, int8
{ amprocfamily => 'brin/integer_minmax_ops', amproclefttype => 'int8',
amprocrighttype => 'int8', amprocnum => '1',
@@ -897,6 +951,58 @@
{ amprocfamily => 'brin/integer_minmax_ops', amproclefttype => 'int4',
amprocrighttype => 'int2', amprocnum => '4', amproc => 'brin_minmax_union' },
+# bloom integer: int2, int4, int8
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int8',
+ amprocrighttype => 'int8', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int8',
+ amprocrighttype => 'int8', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int8',
+ amprocrighttype => 'int8', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int8',
+ amprocrighttype => 'int8', amprocnum => '4', amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int8',
+ amprocrighttype => 'int8', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int8',
+ amprocrighttype => 'int8', amprocnum => '11', amproc => 'hashint8' },
+
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int2',
+ amprocrighttype => 'int2', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int2',
+ amprocrighttype => 'int2', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int2',
+ amprocrighttype => 'int2', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int2',
+ amprocrighttype => 'int2', amprocnum => '4', amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int2',
+ amprocrighttype => 'int2', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int2',
+ amprocrighttype => 'int2', amprocnum => '11', amproc => 'hashint2' },
+
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int4',
+ amprocrighttype => 'int4', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int4',
+ amprocrighttype => 'int4', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int4',
+ amprocrighttype => 'int4', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int4',
+ amprocrighttype => 'int4', amprocnum => '4', amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int4',
+ amprocrighttype => 'int4', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/integer_bloom_ops', amproclefttype => 'int4',
+ amprocrighttype => 'int4', amprocnum => '11', amproc => 'hashint4' },
+
# minmax text
{ amprocfamily => 'brin/text_minmax_ops', amproclefttype => 'text',
amprocrighttype => 'text', amprocnum => '1',
@@ -910,6 +1016,24 @@
{ amprocfamily => 'brin/text_minmax_ops', amproclefttype => 'text',
amprocrighttype => 'text', amprocnum => '4', amproc => 'brin_minmax_union' },
+# bloom text
+{ amprocfamily => 'brin/text_bloom_ops', amproclefttype => 'text',
+ amprocrighttype => 'text', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/text_bloom_ops', amproclefttype => 'text',
+ amprocrighttype => 'text', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/text_bloom_ops', amproclefttype => 'text',
+ amprocrighttype => 'text', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/text_bloom_ops', amproclefttype => 'text',
+ amprocrighttype => 'text', amprocnum => '4', amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/text_bloom_ops', amproclefttype => 'text',
+ amprocrighttype => 'text', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/text_bloom_ops', amproclefttype => 'text',
+ amprocrighttype => 'text', amprocnum => '11', amproc => 'hashtext' },
+
# minmax oid
{ amprocfamily => 'brin/oid_minmax_ops', amproclefttype => 'oid',
amprocrighttype => 'oid', amprocnum => '1', amproc => 'brin_minmax_opcinfo' },
@@ -922,6 +1046,23 @@
{ amprocfamily => 'brin/oid_minmax_ops', amproclefttype => 'oid',
amprocrighttype => 'oid', amprocnum => '4', amproc => 'brin_minmax_union' },
+# bloom oid
+{ amprocfamily => 'brin/oid_bloom_ops', amproclefttype => 'oid',
+ amprocrighttype => 'oid', amprocnum => '1', amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/oid_bloom_ops', amproclefttype => 'oid',
+ amprocrighttype => 'oid', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/oid_bloom_ops', amproclefttype => 'oid',
+ amprocrighttype => 'oid', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/oid_bloom_ops', amproclefttype => 'oid',
+ amprocrighttype => 'oid', amprocnum => '4', amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/oid_bloom_ops', amproclefttype => 'oid',
+ amprocrighttype => 'oid', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/oid_bloom_ops', amproclefttype => 'oid',
+ amprocrighttype => 'oid', amprocnum => '11', amproc => 'hashoid' },
+
# minmax tid
{ amprocfamily => 'brin/tid_minmax_ops', amproclefttype => 'tid',
amprocrighttype => 'tid', amprocnum => '1', amproc => 'brin_minmax_opcinfo' },
@@ -984,6 +1125,45 @@
amprocrighttype => 'float4', amprocnum => '4',
amproc => 'brin_minmax_union' },
+# bloom float
+{ amprocfamily => 'brin/float_bloom_ops', amproclefttype => 'float4',
+ amprocrighttype => 'float4', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/float_bloom_ops', amproclefttype => 'float4',
+ amprocrighttype => 'float4', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/float_bloom_ops', amproclefttype => 'float4',
+ amprocrighttype => 'float4', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/float_bloom_ops', amproclefttype => 'float4',
+ amprocrighttype => 'float4', amprocnum => '4',
+ amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/float_bloom_ops', amproclefttype => 'float4',
+ amprocrighttype => 'float4', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/float_bloom_ops', amproclefttype => 'float4',
+ amprocrighttype => 'float4', amprocnum => '11',
+ amproc => 'hashfloat4' },
+
+{ amprocfamily => 'brin/float_bloom_ops', amproclefttype => 'float8',
+ amprocrighttype => 'float8', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/float_bloom_ops', amproclefttype => 'float8',
+ amprocrighttype => 'float8', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/float_bloom_ops', amproclefttype => 'float8',
+ amprocrighttype => 'float8', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/float_bloom_ops', amproclefttype => 'float8',
+ amprocrighttype => 'float8', amprocnum => '4',
+ amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/float_bloom_ops', amproclefttype => 'float8',
+ amprocrighttype => 'float8', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/float_bloom_ops', amproclefttype => 'float8',
+ amprocrighttype => 'float8', amprocnum => '11',
+ amproc => 'hashfloat8' },
+
# minmax macaddr
{ amprocfamily => 'brin/macaddr_minmax_ops', amproclefttype => 'macaddr',
amprocrighttype => 'macaddr', amprocnum => '1',
@@ -998,6 +1178,26 @@
amprocrighttype => 'macaddr', amprocnum => '4',
amproc => 'brin_minmax_union' },
+# bloom macaddr
+{ amprocfamily => 'brin/macaddr_bloom_ops', amproclefttype => 'macaddr',
+ amprocrighttype => 'macaddr', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/macaddr_bloom_ops', amproclefttype => 'macaddr',
+ amprocrighttype => 'macaddr', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/macaddr_bloom_ops', amproclefttype => 'macaddr',
+ amprocrighttype => 'macaddr', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/macaddr_bloom_ops', amproclefttype => 'macaddr',
+ amprocrighttype => 'macaddr', amprocnum => '4',
+ amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/macaddr_bloom_ops', amproclefttype => 'macaddr',
+ amprocrighttype => 'macaddr', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/macaddr_bloom_ops', amproclefttype => 'macaddr',
+ amprocrighttype => 'macaddr', amprocnum => '11',
+ amproc => 'hashmacaddr' },
+
# minmax macaddr8
{ amprocfamily => 'brin/macaddr8_minmax_ops', amproclefttype => 'macaddr8',
amprocrighttype => 'macaddr8', amprocnum => '1',
@@ -1012,6 +1212,26 @@
amprocrighttype => 'macaddr8', amprocnum => '4',
amproc => 'brin_minmax_union' },
+# bloom macaddr8
+{ amprocfamily => 'brin/macaddr8_bloom_ops', amproclefttype => 'macaddr8',
+ amprocrighttype => 'macaddr8', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/macaddr8_bloom_ops', amproclefttype => 'macaddr8',
+ amprocrighttype => 'macaddr8', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/macaddr8_bloom_ops', amproclefttype => 'macaddr8',
+ amprocrighttype => 'macaddr8', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/macaddr8_bloom_ops', amproclefttype => 'macaddr8',
+ amprocrighttype => 'macaddr8', amprocnum => '4',
+ amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/macaddr8_bloom_ops', amproclefttype => 'macaddr8',
+ amprocrighttype => 'macaddr8', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/macaddr8_bloom_ops', amproclefttype => 'macaddr8',
+ amprocrighttype => 'macaddr8', amprocnum => '11',
+ amproc => 'hashmacaddr8' },
+
# minmax inet
{ amprocfamily => 'brin/network_minmax_ops', amproclefttype => 'inet',
amprocrighttype => 'inet', amprocnum => '1',
@@ -1025,6 +1245,24 @@
{ amprocfamily => 'brin/network_minmax_ops', amproclefttype => 'inet',
amprocrighttype => 'inet', amprocnum => '4', amproc => 'brin_minmax_union' },
+# bloom inet
+{ amprocfamily => 'brin/network_bloom_ops', amproclefttype => 'inet',
+ amprocrighttype => 'inet', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/network_bloom_ops', amproclefttype => 'inet',
+ amprocrighttype => 'inet', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/network_bloom_ops', amproclefttype => 'inet',
+ amprocrighttype => 'inet', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/network_bloom_ops', amproclefttype => 'inet',
+ amprocrighttype => 'inet', amprocnum => '4', amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/network_bloom_ops', amproclefttype => 'inet',
+ amprocrighttype => 'inet', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/network_bloom_ops', amproclefttype => 'inet',
+ amprocrighttype => 'inet', amprocnum => '11', amproc => 'hashinet' },
+
# inclusion inet
{ amprocfamily => 'brin/network_inclusion_ops', amproclefttype => 'inet',
amprocrighttype => 'inet', amprocnum => '1',
@@ -1059,6 +1297,26 @@
amprocrighttype => 'bpchar', amprocnum => '4',
amproc => 'brin_minmax_union' },
+# bloom character
+{ amprocfamily => 'brin/bpchar_bloom_ops', amproclefttype => 'bpchar',
+ amprocrighttype => 'bpchar', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/bpchar_bloom_ops', amproclefttype => 'bpchar',
+ amprocrighttype => 'bpchar', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/bpchar_bloom_ops', amproclefttype => 'bpchar',
+ amprocrighttype => 'bpchar', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/bpchar_bloom_ops', amproclefttype => 'bpchar',
+ amprocrighttype => 'bpchar', amprocnum => '4',
+ amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/bpchar_bloom_ops', amproclefttype => 'bpchar',
+ amprocrighttype => 'bpchar', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/bpchar_bloom_ops', amproclefttype => 'bpchar',
+ amprocrighttype => 'bpchar', amprocnum => '11',
+ amproc => 'hashchar' },
+
# minmax time without time zone
{ amprocfamily => 'brin/time_minmax_ops', amproclefttype => 'time',
amprocrighttype => 'time', amprocnum => '1',
@@ -1072,6 +1330,24 @@
{ amprocfamily => 'brin/time_minmax_ops', amproclefttype => 'time',
amprocrighttype => 'time', amprocnum => '4', amproc => 'brin_minmax_union' },
+# bloom time without time zone
+{ amprocfamily => 'brin/time_bloom_ops', amproclefttype => 'time',
+ amprocrighttype => 'time', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/time_bloom_ops', amproclefttype => 'time',
+ amprocrighttype => 'time', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/time_bloom_ops', amproclefttype => 'time',
+ amprocrighttype => 'time', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/time_bloom_ops', amproclefttype => 'time',
+ amprocrighttype => 'time', amprocnum => '4', amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/time_bloom_ops', amproclefttype => 'time',
+ amprocrighttype => 'time', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/time_bloom_ops', amproclefttype => 'time',
+ amprocrighttype => 'time', amprocnum => '11', amproc => 'time_hash' },
+
# minmax datetime (date, timestamp, timestamptz)
{ amprocfamily => 'brin/datetime_minmax_ops', amproclefttype => 'timestamp',
amprocrighttype => 'timestamp', amprocnum => '1',
@@ -1179,6 +1455,62 @@
amprocrighttype => 'timestamptz', amprocnum => '4',
amproc => 'brin_minmax_union' },
+# bloom datetime (date, timestamp, timestamptz)
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'timestamp',
+ amprocrighttype => 'timestamp', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'timestamp',
+ amprocrighttype => 'timestamp', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'timestamp',
+ amprocrighttype => 'timestamp', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'timestamp',
+ amprocrighttype => 'timestamp', amprocnum => '4',
+ amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'timestamp',
+ amprocrighttype => 'timestamp', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'timestamp',
+ amprocrighttype => 'timestamp', amprocnum => '11',
+ amproc => 'timestamp_hash' },
+
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'timestamptz',
+ amprocrighttype => 'timestamptz', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'timestamptz',
+ amprocrighttype => 'timestamptz', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'timestamptz',
+ amprocrighttype => 'timestamptz', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'timestamptz',
+ amprocrighttype => 'timestamptz', amprocnum => '4',
+ amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'timestamptz',
+ amprocrighttype => 'timestamptz', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'timestamptz',
+ amprocrighttype => 'timestamptz', amprocnum => '11',
+ amproc => 'timestamp_hash' },
+
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'date',
+ amprocrighttype => 'date', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'date',
+ amprocrighttype => 'date', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'date',
+ amprocrighttype => 'date', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'date',
+ amprocrighttype => 'date', amprocnum => '4', amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'date',
+ amprocrighttype => 'date', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/datetime_bloom_ops', amproclefttype => 'date',
+ amprocrighttype => 'date', amprocnum => '11', amproc => 'hashint4' },
+
# minmax interval
{ amprocfamily => 'brin/interval_minmax_ops', amproclefttype => 'interval',
amprocrighttype => 'interval', amprocnum => '1',
@@ -1193,6 +1525,26 @@
amprocrighttype => 'interval', amprocnum => '4',
amproc => 'brin_minmax_union' },
+# bloom interval
+{ amprocfamily => 'brin/interval_bloom_ops', amproclefttype => 'interval',
+ amprocrighttype => 'interval', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/interval_bloom_ops', amproclefttype => 'interval',
+ amprocrighttype => 'interval', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/interval_bloom_ops', amproclefttype => 'interval',
+ amprocrighttype => 'interval', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/interval_bloom_ops', amproclefttype => 'interval',
+ amprocrighttype => 'interval', amprocnum => '4',
+ amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/interval_bloom_ops', amproclefttype => 'interval',
+ amprocrighttype => 'interval', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/interval_bloom_ops', amproclefttype => 'interval',
+ amprocrighttype => 'interval', amprocnum => '11',
+ amproc => 'interval_hash' },
+
# minmax time with time zone
{ amprocfamily => 'brin/timetz_minmax_ops', amproclefttype => 'timetz',
amprocrighttype => 'timetz', amprocnum => '1',
@@ -1207,6 +1559,26 @@
amprocrighttype => 'timetz', amprocnum => '4',
amproc => 'brin_minmax_union' },
+# bloom time with time zone
+{ amprocfamily => 'brin/timetz_bloom_ops', amproclefttype => 'timetz',
+ amprocrighttype => 'timetz', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/timetz_bloom_ops', amproclefttype => 'timetz',
+ amprocrighttype => 'timetz', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/timetz_bloom_ops', amproclefttype => 'timetz',
+ amprocrighttype => 'timetz', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/timetz_bloom_ops', amproclefttype => 'timetz',
+ amprocrighttype => 'timetz', amprocnum => '4',
+ amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/timetz_bloom_ops', amproclefttype => 'timetz',
+ amprocrighttype => 'timetz', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/timetz_bloom_ops', amproclefttype => 'timetz',
+ amprocrighttype => 'timetz', amprocnum => '11',
+ amproc => 'timetz_hash' },
+
# minmax bit
{ amprocfamily => 'brin/bit_minmax_ops', amproclefttype => 'bit',
amprocrighttype => 'bit', amprocnum => '1', amproc => 'brin_minmax_opcinfo' },
@@ -1247,6 +1619,26 @@
amprocrighttype => 'numeric', amprocnum => '4',
amproc => 'brin_minmax_union' },
+# bloom numeric
+{ amprocfamily => 'brin/numeric_bloom_ops', amproclefttype => 'numeric',
+ amprocrighttype => 'numeric', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/numeric_bloom_ops', amproclefttype => 'numeric',
+ amprocrighttype => 'numeric', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/numeric_bloom_ops', amproclefttype => 'numeric',
+ amprocrighttype => 'numeric', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/numeric_bloom_ops', amproclefttype => 'numeric',
+ amprocrighttype => 'numeric', amprocnum => '4',
+ amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/numeric_bloom_ops', amproclefttype => 'numeric',
+ amprocrighttype => 'numeric', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/numeric_bloom_ops', amproclefttype => 'numeric',
+ amprocrighttype => 'numeric', amprocnum => '11',
+ amproc => 'hash_numeric' },
+
# minmax uuid
{ amprocfamily => 'brin/uuid_minmax_ops', amproclefttype => 'uuid',
amprocrighttype => 'uuid', amprocnum => '1',
@@ -1260,6 +1652,24 @@
{ amprocfamily => 'brin/uuid_minmax_ops', amproclefttype => 'uuid',
amprocrighttype => 'uuid', amprocnum => '4', amproc => 'brin_minmax_union' },
+# bloom uuid
+{ amprocfamily => 'brin/uuid_bloom_ops', amproclefttype => 'uuid',
+ amprocrighttype => 'uuid', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/uuid_bloom_ops', amproclefttype => 'uuid',
+ amprocrighttype => 'uuid', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/uuid_bloom_ops', amproclefttype => 'uuid',
+ amprocrighttype => 'uuid', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/uuid_bloom_ops', amproclefttype => 'uuid',
+ amprocrighttype => 'uuid', amprocnum => '4', amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/uuid_bloom_ops', amproclefttype => 'uuid',
+ amprocrighttype => 'uuid', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/uuid_bloom_ops', amproclefttype => 'uuid',
+ amprocrighttype => 'uuid', amprocnum => '11', amproc => 'uuid_hash' },
+
# inclusion range types
{ amprocfamily => 'brin/range_inclusion_ops', amproclefttype => 'anyrange',
amprocrighttype => 'anyrange', amprocnum => '1',
@@ -1295,6 +1705,26 @@
amprocrighttype => 'pg_lsn', amprocnum => '4',
amproc => 'brin_minmax_union' },
+# bloom pg_lsn
+{ amprocfamily => 'brin/pg_lsn_bloom_ops', amproclefttype => 'pg_lsn',
+ amprocrighttype => 'pg_lsn', amprocnum => '1',
+ amproc => 'brin_bloom_opcinfo' },
+{ amprocfamily => 'brin/pg_lsn_bloom_ops', amproclefttype => 'pg_lsn',
+ amprocrighttype => 'pg_lsn', amprocnum => '2',
+ amproc => 'brin_bloom_add_value' },
+{ amprocfamily => 'brin/pg_lsn_bloom_ops', amproclefttype => 'pg_lsn',
+ amprocrighttype => 'pg_lsn', amprocnum => '3',
+ amproc => 'brin_bloom_consistent' },
+{ amprocfamily => 'brin/pg_lsn_bloom_ops', amproclefttype => 'pg_lsn',
+ amprocrighttype => 'pg_lsn', amprocnum => '4',
+ amproc => 'brin_bloom_union' },
+{ amprocfamily => 'brin/pg_lsn_bloom_ops', amproclefttype => 'pg_lsn',
+ amprocrighttype => 'pg_lsn', amprocnum => '5',
+ amproc => 'brin_bloom_options' },
+{ amprocfamily => 'brin/pg_lsn_bloom_ops', amproclefttype => 'pg_lsn',
+ amprocrighttype => 'pg_lsn', amprocnum => '11',
+ amproc => 'pg_lsn_hash' },
+
# inclusion box
{ amprocfamily => 'brin/box_inclusion_ops', amproclefttype => 'box',
amprocrighttype => 'box', amprocnum => '1',
diff --git a/src/include/catalog/pg_opclass.dat b/src/include/catalog/pg_opclass.dat
index f2342bb328..16d6111ab6 100644
--- a/src/include/catalog/pg_opclass.dat
+++ b/src/include/catalog/pg_opclass.dat
@@ -257,67 +257,127 @@
{ opcmethod => 'brin', opcname => 'bytea_minmax_ops',
opcfamily => 'brin/bytea_minmax_ops', opcintype => 'bytea',
opckeytype => 'bytea' },
+{ opcmethod => 'brin', opcname => 'bytea_bloom_ops',
+ opcfamily => 'brin/bytea_bloom_ops', opcintype => 'bytea',
+ opckeytype => 'bytea', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'char_minmax_ops',
opcfamily => 'brin/char_minmax_ops', opcintype => 'char',
opckeytype => 'char' },
+{ opcmethod => 'brin', opcname => 'char_bloom_ops',
+ opcfamily => 'brin/char_bloom_ops', opcintype => 'char',
+ opckeytype => 'char', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'name_minmax_ops',
opcfamily => 'brin/name_minmax_ops', opcintype => 'name',
opckeytype => 'name' },
+{ opcmethod => 'brin', opcname => 'name_bloom_ops',
+ opcfamily => 'brin/name_bloom_ops', opcintype => 'name',
+ opckeytype => 'name', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'int8_minmax_ops',
opcfamily => 'brin/integer_minmax_ops', opcintype => 'int8',
opckeytype => 'int8' },
+{ opcmethod => 'brin', opcname => 'int8_bloom_ops',
+ opcfamily => 'brin/integer_bloom_ops', opcintype => 'int8',
+ opckeytype => 'int8', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'int2_minmax_ops',
opcfamily => 'brin/integer_minmax_ops', opcintype => 'int2',
opckeytype => 'int2' },
+{ opcmethod => 'brin', opcname => 'int2_bloom_ops',
+ opcfamily => 'brin/integer_bloom_ops', opcintype => 'int2',
+ opckeytype => 'int2', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'int4_minmax_ops',
opcfamily => 'brin/integer_minmax_ops', opcintype => 'int4',
opckeytype => 'int4' },
+{ opcmethod => 'brin', opcname => 'int4_bloom_ops',
+ opcfamily => 'brin/integer_bloom_ops', opcintype => 'int4',
+ opckeytype => 'int4', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'text_minmax_ops',
opcfamily => 'brin/text_minmax_ops', opcintype => 'text',
opckeytype => 'text' },
+{ opcmethod => 'brin', opcname => 'text_bloom_ops',
+ opcfamily => 'brin/text_bloom_ops', opcintype => 'text',
+ opckeytype => 'text', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'oid_minmax_ops',
opcfamily => 'brin/oid_minmax_ops', opcintype => 'oid', opckeytype => 'oid' },
+{ opcmethod => 'brin', opcname => 'oid_bloom_ops',
+ opcfamily => 'brin/oid_bloom_ops', opcintype => 'oid',
+ opckeytype => 'oid', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'tid_minmax_ops',
opcfamily => 'brin/tid_minmax_ops', opcintype => 'tid', opckeytype => 'tid' },
{ opcmethod => 'brin', opcname => 'float4_minmax_ops',
opcfamily => 'brin/float_minmax_ops', opcintype => 'float4',
opckeytype => 'float4' },
+{ opcmethod => 'brin', opcname => 'float4_bloom_ops',
+ opcfamily => 'brin/float_bloom_ops', opcintype => 'float4',
+ opckeytype => 'float4', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'float8_minmax_ops',
opcfamily => 'brin/float_minmax_ops', opcintype => 'float8',
opckeytype => 'float8' },
+{ opcmethod => 'brin', opcname => 'float8_bloom_ops',
+ opcfamily => 'brin/float_bloom_ops', opcintype => 'float8',
+ opckeytype => 'float8', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'macaddr_minmax_ops',
opcfamily => 'brin/macaddr_minmax_ops', opcintype => 'macaddr',
opckeytype => 'macaddr' },
+{ opcmethod => 'brin', opcname => 'macaddr_bloom_ops',
+ opcfamily => 'brin/macaddr_bloom_ops', opcintype => 'macaddr',
+ opckeytype => 'macaddr', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'macaddr8_minmax_ops',
opcfamily => 'brin/macaddr8_minmax_ops', opcintype => 'macaddr8',
opckeytype => 'macaddr8' },
+{ opcmethod => 'brin', opcname => 'macaddr8_bloom_ops',
+ opcfamily => 'brin/macaddr8_bloom_ops', opcintype => 'macaddr8',
+ opckeytype => 'macaddr8', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'inet_minmax_ops',
opcfamily => 'brin/network_minmax_ops', opcintype => 'inet',
opcdefault => 'f', opckeytype => 'inet' },
+{ opcmethod => 'brin', opcname => 'inet_bloom_ops',
+ opcfamily => 'brin/network_bloom_ops', opcintype => 'inet',
+ opcdefault => 'f', opckeytype => 'inet', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'inet_inclusion_ops',
opcfamily => 'brin/network_inclusion_ops', opcintype => 'inet',
opckeytype => 'inet' },
{ opcmethod => 'brin', opcname => 'bpchar_minmax_ops',
opcfamily => 'brin/bpchar_minmax_ops', opcintype => 'bpchar',
opckeytype => 'bpchar' },
+{ opcmethod => 'brin', opcname => 'bpchar_bloom_ops',
+ opcfamily => 'brin/bpchar_bloom_ops', opcintype => 'bpchar',
+ opckeytype => 'bpchar', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'time_minmax_ops',
opcfamily => 'brin/time_minmax_ops', opcintype => 'time',
opckeytype => 'time' },
+{ opcmethod => 'brin', opcname => 'time_bloom_ops',
+ opcfamily => 'brin/time_bloom_ops', opcintype => 'time',
+ opckeytype => 'time', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'date_minmax_ops',
opcfamily => 'brin/datetime_minmax_ops', opcintype => 'date',
opckeytype => 'date' },
+{ opcmethod => 'brin', opcname => 'date_bloom_ops',
+ opcfamily => 'brin/datetime_bloom_ops', opcintype => 'date',
+ opckeytype => 'date', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'timestamp_minmax_ops',
opcfamily => 'brin/datetime_minmax_ops', opcintype => 'timestamp',
opckeytype => 'timestamp' },
+{ opcmethod => 'brin', opcname => 'timestamp_bloom_ops',
+ opcfamily => 'brin/datetime_bloom_ops', opcintype => 'timestamp',
+ opckeytype => 'timestamp', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'timestamptz_minmax_ops',
opcfamily => 'brin/datetime_minmax_ops', opcintype => 'timestamptz',
opckeytype => 'timestamptz' },
+{ opcmethod => 'brin', opcname => 'timestamptz_bloom_ops',
+ opcfamily => 'brin/datetime_bloom_ops', opcintype => 'timestamptz',
+ opckeytype => 'timestamptz', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'interval_minmax_ops',
opcfamily => 'brin/interval_minmax_ops', opcintype => 'interval',
opckeytype => 'interval' },
+{ opcmethod => 'brin', opcname => 'interval_bloom_ops',
+ opcfamily => 'brin/interval_bloom_ops', opcintype => 'interval',
+ opckeytype => 'interval', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'timetz_minmax_ops',
opcfamily => 'brin/timetz_minmax_ops', opcintype => 'timetz',
opckeytype => 'timetz' },
+{ opcmethod => 'brin', opcname => 'timetz_bloom_ops',
+ opcfamily => 'brin/timetz_bloom_ops', opcintype => 'timetz',
+ opckeytype => 'timetz', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'bit_minmax_ops',
opcfamily => 'brin/bit_minmax_ops', opcintype => 'bit', opckeytype => 'bit' },
{ opcmethod => 'brin', opcname => 'varbit_minmax_ops',
@@ -326,18 +386,27 @@
{ opcmethod => 'brin', opcname => 'numeric_minmax_ops',
opcfamily => 'brin/numeric_minmax_ops', opcintype => 'numeric',
opckeytype => 'numeric' },
+{ opcmethod => 'brin', opcname => 'numeric_bloom_ops',
+ opcfamily => 'brin/numeric_bloom_ops', opcintype => 'numeric',
+ opckeytype => 'numeric', opcdefault => 'f' },
# no brin opclass for record, anyarray
{ opcmethod => 'brin', opcname => 'uuid_minmax_ops',
opcfamily => 'brin/uuid_minmax_ops', opcintype => 'uuid',
opckeytype => 'uuid' },
+{ opcmethod => 'brin', opcname => 'uuid_bloom_ops',
+ opcfamily => 'brin/uuid_bloom_ops', opcintype => 'uuid',
+ opckeytype => 'uuid', opcdefault => 'f' },
{ opcmethod => 'brin', opcname => 'range_inclusion_ops',
opcfamily => 'brin/range_inclusion_ops', opcintype => 'anyrange',
opckeytype => 'anyrange' },
{ opcmethod => 'brin', opcname => 'pg_lsn_minmax_ops',
opcfamily => 'brin/pg_lsn_minmax_ops', opcintype => 'pg_lsn',
opckeytype => 'pg_lsn' },
+{ opcmethod => 'brin', opcname => 'pg_lsn_bloom_ops',
+ opcfamily => 'brin/pg_lsn_bloom_ops', opcintype => 'pg_lsn',
+ opckeytype => 'pg_lsn', opcdefault => 'f' },
# no brin opclass for enum, tsvector, tsquery, jsonb
diff --git a/src/include/catalog/pg_opfamily.dat b/src/include/catalog/pg_opfamily.dat
index cf0fb325b3..7f733aeab1 100644
--- a/src/include/catalog/pg_opfamily.dat
+++ b/src/include/catalog/pg_opfamily.dat
@@ -180,50 +180,86 @@
opfmethod => 'gin', opfname => 'jsonb_path_ops' },
{ oid => '4054',
opfmethod => 'brin', opfname => 'integer_minmax_ops' },
+{ oid => '8100',
+ opfmethod => 'brin', opfname => 'integer_bloom_ops' },
{ oid => '4055',
opfmethod => 'brin', opfname => 'numeric_minmax_ops' },
{ oid => '4056',
opfmethod => 'brin', opfname => 'text_minmax_ops' },
+{ oid => '8101',
+ opfmethod => 'brin', opfname => 'text_bloom_ops' },
+{ oid => '8102',
+ opfmethod => 'brin', opfname => 'numeric_bloom_ops' },
{ oid => '4058',
opfmethod => 'brin', opfname => 'timetz_minmax_ops' },
+{ oid => '8103',
+ opfmethod => 'brin', opfname => 'timetz_bloom_ops' },
{ oid => '4059',
opfmethod => 'brin', opfname => 'datetime_minmax_ops' },
+{ oid => '8104',
+ opfmethod => 'brin', opfname => 'datetime_bloom_ops' },
{ oid => '4062',
opfmethod => 'brin', opfname => 'char_minmax_ops' },
+{ oid => '8105',
+ opfmethod => 'brin', opfname => 'char_bloom_ops' },
{ oid => '4064',
opfmethod => 'brin', opfname => 'bytea_minmax_ops' },
+{ oid => '8106',
+ opfmethod => 'brin', opfname => 'bytea_bloom_ops' },
{ oid => '4065',
opfmethod => 'brin', opfname => 'name_minmax_ops' },
+{ oid => '8107',
+ opfmethod => 'brin', opfname => 'name_bloom_ops' },
{ oid => '4068',
opfmethod => 'brin', opfname => 'oid_minmax_ops' },
+{ oid => '8108',
+ opfmethod => 'brin', opfname => 'oid_bloom_ops' },
{ oid => '4069',
opfmethod => 'brin', opfname => 'tid_minmax_ops' },
{ oid => '4070',
opfmethod => 'brin', opfname => 'float_minmax_ops' },
+{ oid => '8109',
+ opfmethod => 'brin', opfname => 'float_bloom_ops' },
{ oid => '4074',
opfmethod => 'brin', opfname => 'macaddr_minmax_ops' },
+{ oid => '8110',
+ opfmethod => 'brin', opfname => 'macaddr_bloom_ops' },
{ oid => '4109',
opfmethod => 'brin', opfname => 'macaddr8_minmax_ops' },
+{ oid => '8111',
+ opfmethod => 'brin', opfname => 'macaddr8_bloom_ops' },
{ oid => '4075',
opfmethod => 'brin', opfname => 'network_minmax_ops' },
{ oid => '4102',
opfmethod => 'brin', opfname => 'network_inclusion_ops' },
+{ oid => '8112',
+ opfmethod => 'brin', opfname => 'network_bloom_ops' },
{ oid => '4076',
opfmethod => 'brin', opfname => 'bpchar_minmax_ops' },
+{ oid => '8113',
+ opfmethod => 'brin', opfname => 'bpchar_bloom_ops' },
{ oid => '4077',
opfmethod => 'brin', opfname => 'time_minmax_ops' },
+{ oid => '8114',
+ opfmethod => 'brin', opfname => 'time_bloom_ops' },
{ oid => '4078',
opfmethod => 'brin', opfname => 'interval_minmax_ops' },
+{ oid => '8115',
+ opfmethod => 'brin', opfname => 'interval_bloom_ops' },
{ oid => '4079',
opfmethod => 'brin', opfname => 'bit_minmax_ops' },
{ oid => '4080',
opfmethod => 'brin', opfname => 'varbit_minmax_ops' },
{ oid => '4081',
opfmethod => 'brin', opfname => 'uuid_minmax_ops' },
+{ oid => '8116',
+ opfmethod => 'brin', opfname => 'uuid_bloom_ops' },
{ oid => '4103',
opfmethod => 'brin', opfname => 'range_inclusion_ops' },
{ oid => '4082',
opfmethod => 'brin', opfname => 'pg_lsn_minmax_ops' },
+{ oid => '8117',
+ opfmethod => 'brin', opfname => 'pg_lsn_bloom_ops' },
{ oid => '4104',
opfmethod => 'brin', opfname => 'box_inclusion_ops' },
{ oid => '5000',
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 018a6490fb..5336aa7b9c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -8109,6 +8109,26 @@
proargtypes => 'internal internal internal',
prosrc => 'brin_inclusion_union' },
+# BRIN bloom
+{ oid => '8118', descr => 'BRIN bloom support',
+ proname => 'brin_bloom_opcinfo', prorettype => 'internal',
+ proargtypes => 'internal', prosrc => 'brin_bloom_opcinfo' },
+{ oid => '8119', descr => 'BRIN bloom support',
+ proname => 'brin_bloom_add_value', prorettype => 'bool',
+ proargtypes => 'internal internal internal internal',
+ prosrc => 'brin_bloom_add_value' },
+{ oid => '8120', descr => 'BRIN bloom support',
+ proname => 'brin_bloom_consistent', prorettype => 'bool',
+ proargtypes => 'internal internal internal int4',
+ prosrc => 'brin_bloom_consistent' },
+{ oid => '8121', descr => 'BRIN bloom support',
+ proname => 'brin_bloom_union', prorettype => 'bool',
+ proargtypes => 'internal internal internal',
+ prosrc => 'brin_bloom_union' },
+{ oid => '8122', descr => 'BRIN bloom support',
+ proname => 'brin_bloom_options', prorettype => 'void', proisstrict => 'f',
+ proargtypes => 'internal', prosrc => 'brin_bloom_options' },
+
# userlock replacements
{ oid => '2880', descr => 'obtain exclusive advisory lock',
proname => 'pg_advisory_lock', provolatile => 'v', proparallel => 'r',
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
new file mode 100644
index 0000000000..d58a4a483c
--- /dev/null
+++ b/src/test/regress/expected/brin_bloom.out
@@ -0,0 +1,456 @@
+CREATE TABLE brintest_bloom (byteacol bytea,
+ charcol "char",
+ namecol name,
+ int8col bigint,
+ int2col smallint,
+ int4col integer,
+ textcol text,
+ oidcol oid,
+ float4col real,
+ float8col double precision,
+ macaddrcol macaddr,
+ inetcol inet,
+ cidrcol cidr,
+ bpcharcol character,
+ datecol date,
+ timecol time without time zone,
+ timestampcol timestamp without time zone,
+ timestamptzcol timestamp with time zone,
+ intervalcol interval,
+ timetzcol time with time zone,
+ numericcol numeric,
+ uuidcol uuid,
+ lsncol pg_lsn
+) WITH (fillfactor=10);
+INSERT INTO brintest_bloom SELECT
+ repeat(stringu1, 8)::bytea,
+ substr(stringu1, 1, 1)::"char",
+ stringu1::name, 142857 * tenthous,
+ thousand,
+ twothousand,
+ repeat(stringu1, 8),
+ unique1::oid,
+ (four + 1.0)/(hundred+1),
+ odd::float8 / (tenthous + 1),
+ format('%s:00:%s:00:%s:00', to_hex(odd), to_hex(even), to_hex(hundred))::macaddr,
+ inet '10.2.3.4/24' + tenthous,
+ cidr '10.2.3/24' + tenthous,
+ substr(stringu1, 1, 1)::bpchar,
+ date '1995-08-15' + tenthous,
+ time '01:20:30' + thousand * interval '18.5 second',
+ timestamp '1942-07-23 03:05:09' + tenthous * interval '36.38 hours',
+ timestamptz '1972-10-10 03:00' + thousand * interval '1 hour',
+ justify_days(justify_hours(tenthous * interval '12 minutes')),
+ timetz '01:30:20+02' + hundred * interval '15 seconds',
+ tenthous::numeric(36,30) * fivethous * even / (hundred + 1),
+ format('%s%s-%s-%s-%s-%s%s%s', to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'))::uuid,
+ format('%s/%s%s', odd, even, tenthous)::pg_lsn
+FROM tenk1 ORDER BY unique2 LIMIT 100;
+-- throw in some NULL's and different values
+INSERT INTO brintest_bloom (inetcol, cidrcol) SELECT
+ inet 'fe80::6e40:8ff:fea9:8c46' + tenthous,
+ cidr 'fe80::6e40:8ff:fea9:8c46' + tenthous
+FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
+-- test bloom specific index options
+-- ndistinct must be >= -1.0
+CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
+ byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
+);
+ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
+DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
+-- false_positive_rate must be between 0.001 and 1.0
+CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
+ byteacol bytea_bloom_ops(false_positive_rate = 0.0)
+);
+ERROR: value 0.0 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.001000" and "1.000000".
+CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
+ byteacol bytea_bloom_ops(false_positive_rate = 1.01)
+);
+ERROR: value 1.01 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.001000" and "1.000000".
+CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
+ byteacol bytea_bloom_ops,
+ charcol char_bloom_ops,
+ namecol name_bloom_ops,
+ int8col int8_bloom_ops,
+ int2col int2_bloom_ops,
+ int4col int4_bloom_ops,
+ textcol text_bloom_ops,
+ oidcol oid_bloom_ops,
+ float4col float4_bloom_ops,
+ float8col float8_bloom_ops,
+ macaddrcol macaddr_bloom_ops,
+ inetcol inet_bloom_ops,
+ cidrcol inet_bloom_ops,
+ bpcharcol bpchar_bloom_ops,
+ datecol date_bloom_ops,
+ timecol time_bloom_ops,
+ timestampcol timestamp_bloom_ops,
+ timestamptzcol timestamptz_bloom_ops,
+ intervalcol interval_bloom_ops,
+ timetzcol timetz_bloom_ops,
+ numericcol numeric_bloom_ops,
+ uuidcol uuid_bloom_ops,
+ lsncol pg_lsn_bloom_ops
+) with (pages_per_range = 1);
+CREATE TABLE brinopers_bloom (colname name, typ text,
+ op text[], value text[], matches int[],
+ check (cardinality(op) = cardinality(value)),
+ check (cardinality(op) = cardinality(matches)));
+INSERT INTO brinopers_bloom VALUES
+ ('byteacol', 'bytea',
+ '{=}',
+ '{BNAAAABNAAAABNAAAABNAAAABNAAAABNAAAABNAAAABNAAAA}',
+ '{1}'),
+ ('charcol', '"char"',
+ '{=}',
+ '{M}',
+ '{6}'),
+ ('namecol', 'name',
+ '{=}',
+ '{MAAAAA}',
+ '{2}'),
+ ('int2col', 'int2',
+ '{=}',
+ '{800}',
+ '{1}'),
+ ('int2col', 'int4',
+ '{=}',
+ '{800}',
+ '{1}'),
+ ('int2col', 'int8',
+ '{=}',
+ '{800}',
+ '{1}'),
+ ('int4col', 'int2',
+ '{=}',
+ '{800}',
+ '{1}'),
+ ('int4col', 'int4',
+ '{=}',
+ '{800}',
+ '{1}'),
+ ('int4col', 'int8',
+ '{=}',
+ '{800}',
+ '{1}'),
+ ('int8col', 'int8',
+ '{=}',
+ '{1257141600}',
+ '{1}'),
+ ('textcol', 'text',
+ '{=}',
+ '{BNAAAABNAAAABNAAAABNAAAABNAAAABNAAAABNAAAABNAAAA}',
+ '{1}'),
+ ('oidcol', 'oid',
+ '{=}',
+ '{8800}',
+ '{1}'),
+ ('float4col', 'float4',
+ '{=}',
+ '{1}',
+ '{4}'),
+ ('float4col', 'float8',
+ '{=}',
+ '{1}',
+ '{4}'),
+ ('float8col', 'float4',
+ '{=}',
+ '{0}',
+ '{1}'),
+ ('float8col', 'float8',
+ '{=}',
+ '{0}',
+ '{1}'),
+ ('macaddrcol', 'macaddr',
+ '{=}',
+ '{2c:00:2d:00:16:00}',
+ '{2}'),
+ ('inetcol', 'inet',
+ '{=}',
+ '{10.2.14.231/24}',
+ '{1}'),
+ ('inetcol', 'cidr',
+ '{=}',
+ '{fe80::6e40:8ff:fea9:8c46}',
+ '{1}'),
+ ('cidrcol', 'inet',
+ '{=}',
+ '{10.2.14/24}',
+ '{2}'),
+ ('cidrcol', 'inet',
+ '{=}',
+ '{fe80::6e40:8ff:fea9:8c46}',
+ '{1}'),
+ ('cidrcol', 'cidr',
+ '{=}',
+ '{10.2.14/24}',
+ '{2}'),
+ ('cidrcol', 'cidr',
+ '{=}',
+ '{fe80::6e40:8ff:fea9:8c46}',
+ '{1}'),
+ ('bpcharcol', 'bpchar',
+ '{=}',
+ '{W}',
+ '{6}'),
+ ('datecol', 'date',
+ '{=}',
+ '{2009-12-01}',
+ '{1}'),
+ ('timecol', 'time',
+ '{=}',
+ '{02:28:57}',
+ '{1}'),
+ ('timestampcol', 'timestamp',
+ '{=}',
+ '{1964-03-24 19:26:45}',
+ '{1}'),
+ ('timestampcol', 'timestamptz',
+ '{=}',
+ '{1964-03-24 19:26:45}',
+ '{1}'),
+ ('timestamptzcol', 'timestamptz',
+ '{=}',
+ '{1972-10-19 09:00:00-07}',
+ '{1}'),
+ ('intervalcol', 'interval',
+ '{=}',
+ '{1 mons 13 days 12:24}',
+ '{1}'),
+ ('timetzcol', 'timetz',
+ '{=}',
+ '{01:35:50+02}',
+ '{2}'),
+ ('numericcol', 'numeric',
+ '{=}',
+ '{2268164.347826086956521739130434782609}',
+ '{1}'),
+ ('uuidcol', 'uuid',
+ '{=}',
+ '{52225222-5222-5222-5222-522252225222}',
+ '{1}'),
+ ('lsncol', 'pg_lsn',
+ '{=, IS, IS NOT}',
+ '{44/455222, NULL, NULL}',
+ '{1, 25, 100}');
+DO $x$
+DECLARE
+ r record;
+ r2 record;
+ cond text;
+ idx_ctids tid[];
+ ss_ctids tid[];
+ count int;
+ plan_ok bool;
+ plan_line text;
+BEGIN
+ FOR r IN SELECT colname, oper, typ, value[ordinality], matches[ordinality] FROM brinopers_bloom, unnest(op) WITH ORDINALITY AS oper LOOP
+
+ -- prepare the condition
+ IF r.value IS NULL THEN
+ cond := format('%I %s %L', r.colname, r.oper, r.value);
+ ELSE
+ cond := format('%I %s %L::%s', r.colname, r.oper, r.value, r.typ);
+ END IF;
+
+ -- run the query using the brin index
+ SET enable_seqscan = 0;
+ SET enable_bitmapscan = 1;
+
+ plan_ok := false;
+ FOR plan_line IN EXECUTE format($y$EXPLAIN SELECT array_agg(ctid) FROM brintest_bloom WHERE %s $y$, cond) LOOP
+ IF plan_line LIKE '%Bitmap Heap Scan on brintest_bloom%' THEN
+ plan_ok := true;
+ END IF;
+ END LOOP;
+ IF NOT plan_ok THEN
+ RAISE WARNING 'did not get bitmap indexscan plan for %', r;
+ END IF;
+
+ EXECUTE format($y$SELECT array_agg(ctid) FROM brintest_bloom WHERE %s $y$, cond)
+ INTO idx_ctids;
+
+ -- run the query using a seqscan
+ SET enable_seqscan = 1;
+ SET enable_bitmapscan = 0;
+
+ plan_ok := false;
+ FOR plan_line IN EXECUTE format($y$EXPLAIN SELECT array_agg(ctid) FROM brintest_bloom WHERE %s $y$, cond) LOOP
+ IF plan_line LIKE '%Seq Scan on brintest_bloom%' THEN
+ plan_ok := true;
+ END IF;
+ END LOOP;
+ IF NOT plan_ok THEN
+ RAISE WARNING 'did not get seqscan plan for %', r;
+ END IF;
+
+ EXECUTE format($y$SELECT array_agg(ctid) FROM brintest_bloom WHERE %s $y$, cond)
+ INTO ss_ctids;
+
+ -- make sure both return the same results
+ count := array_length(idx_ctids, 1);
+
+ IF NOT (count = array_length(ss_ctids, 1) AND
+ idx_ctids @> ss_ctids AND
+ idx_ctids <@ ss_ctids) THEN
+ -- report the results of each scan to make the differences obvious
+ RAISE WARNING 'something not right in %: count %', r, count;
+ SET enable_seqscan = 1;
+ SET enable_bitmapscan = 0;
+ FOR r2 IN EXECUTE 'SELECT ' || r.colname || ' FROM brintest_bloom WHERE ' || cond LOOP
+ RAISE NOTICE 'seqscan: %', r2;
+ END LOOP;
+
+ SET enable_seqscan = 0;
+ SET enable_bitmapscan = 1;
+ FOR r2 IN EXECUTE 'SELECT ' || r.colname || ' FROM brintest_bloom WHERE ' || cond LOOP
+ RAISE NOTICE 'bitmapscan: %', r2;
+ END LOOP;
+ END IF;
+
+ -- make sure we found expected number of matches
+ IF count != r.matches THEN RAISE WARNING 'unexpected number of results % for %', count, r; END IF;
+ END LOOP;
+END;
+$x$;
+RESET enable_seqscan;
+RESET enable_bitmapscan;
+INSERT INTO brintest_bloom SELECT
+ repeat(stringu1, 42)::bytea,
+ substr(stringu1, 1, 1)::"char",
+ stringu1::name, 142857 * tenthous,
+ thousand,
+ twothousand,
+ repeat(stringu1, 42),
+ unique1::oid,
+ (four + 1.0)/(hundred+1),
+ odd::float8 / (tenthous + 1),
+ format('%s:00:%s:00:%s:00', to_hex(odd), to_hex(even), to_hex(hundred))::macaddr,
+ inet '10.2.3.4' + tenthous,
+ cidr '10.2.3/24' + tenthous,
+ substr(stringu1, 1, 1)::bpchar,
+ date '1995-08-15' + tenthous,
+ time '01:20:30' + thousand * interval '18.5 second',
+ timestamp '1942-07-23 03:05:09' + tenthous * interval '36.38 hours',
+ timestamptz '1972-10-10 03:00' + thousand * interval '1 hour',
+ justify_days(justify_hours(tenthous * interval '12 minutes')),
+ timetz '01:30:20' + hundred * interval '15 seconds',
+ tenthous::numeric(36,30) * fivethous * even / (hundred + 1),
+ format('%s%s-%s-%s-%s-%s%s%s', to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'))::uuid,
+ format('%s/%s%s', odd, even, tenthous)::pg_lsn
+FROM tenk1 ORDER BY unique2 LIMIT 5 OFFSET 5;
+SELECT brin_desummarize_range('brinidx_bloom', 0);
+ brin_desummarize_range
+------------------------
+
+(1 row)
+
+VACUUM brintest_bloom; -- force a summarization cycle in brinidx
+UPDATE brintest_bloom SET int8col = int8col * int4col;
+UPDATE brintest_bloom SET textcol = '' WHERE textcol IS NOT NULL;
+-- Tests for brin_summarize_new_values
+SELECT brin_summarize_new_values('brintest_bloom'); -- error, not an index
+ERROR: "brintest_bloom" is not an index
+SELECT brin_summarize_new_values('tenk1_unique1'); -- error, not a BRIN index
+ERROR: "tenk1_unique1" is not a BRIN index
+SELECT brin_summarize_new_values('brinidx_bloom'); -- ok, no change expected
+ brin_summarize_new_values
+---------------------------
+ 0
+(1 row)
+
+-- Tests for brin_desummarize_range
+SELECT brin_desummarize_range('brinidx_bloom', -1); -- error, invalid range
+ERROR: block number out of range: -1
+SELECT brin_desummarize_range('brinidx_bloom', 0);
+ brin_desummarize_range
+------------------------
+
+(1 row)
+
+SELECT brin_desummarize_range('brinidx_bloom', 0);
+ brin_desummarize_range
+------------------------
+
+(1 row)
+
+SELECT brin_desummarize_range('brinidx_bloom', 100000000);
+ brin_desummarize_range
+------------------------
+
+(1 row)
+
+-- Test brin_summarize_range
+CREATE TABLE brin_summarize_bloom (
+ value int
+) WITH (fillfactor=10, autovacuum_enabled=false);
+CREATE INDEX brin_summarize_bloom_idx ON brin_summarize_bloom USING brin (value) WITH (pages_per_range=2);
+-- Fill a few pages
+DO $$
+DECLARE curtid tid;
+BEGIN
+ LOOP
+ INSERT INTO brin_summarize_bloom VALUES (1) RETURNING ctid INTO curtid;
+ EXIT WHEN curtid > tid '(2, 0)';
+ END LOOP;
+END;
+$$;
+-- summarize one range
+SELECT brin_summarize_range('brin_summarize_bloom_idx', 0);
+ brin_summarize_range
+----------------------
+ 0
+(1 row)
+
+-- nothing: already summarized
+SELECT brin_summarize_range('brin_summarize_bloom_idx', 1);
+ brin_summarize_range
+----------------------
+ 0
+(1 row)
+
+-- summarize one range
+SELECT brin_summarize_range('brin_summarize_bloom_idx', 2);
+ brin_summarize_range
+----------------------
+ 1
+(1 row)
+
+-- nothing: page doesn't exist in table
+SELECT brin_summarize_range('brin_summarize_bloom_idx', 4294967295);
+ brin_summarize_range
+----------------------
+ 0
+(1 row)
+
+-- invalid block number values
+SELECT brin_summarize_range('brin_summarize_bloom_idx', -1);
+ERROR: block number out of range: -1
+SELECT brin_summarize_range('brin_summarize_bloom_idx', 4294967296);
+ERROR: block number out of range: 4294967296
+-- test brin cost estimates behave sanely based on correlation of values
+CREATE TABLE brin_test_bloom (a INT, b INT);
+INSERT INTO brin_test_bloom SELECT x/100,x%100 FROM generate_series(1,10000) x(x);
+CREATE INDEX brin_test_bloom_a_idx ON brin_test_bloom USING brin (a) WITH (pages_per_range = 2);
+CREATE INDEX brin_test_bloom_b_idx ON brin_test_bloom USING brin (b) WITH (pages_per_range = 2);
+VACUUM ANALYZE brin_test_bloom;
+-- Ensure brin index is used when columns are perfectly correlated
+EXPLAIN (COSTS OFF) SELECT * FROM brin_test_bloom WHERE a = 1;
+ QUERY PLAN
+--------------------------------------------------
+ Bitmap Heap Scan on brin_test_bloom
+ Recheck Cond: (a = 1)
+ -> Bitmap Index Scan on brin_test_bloom_a_idx
+ Index Cond: (a = 1)
+(4 rows)
+
+-- Ensure brin index is not used when values are not correlated
+EXPLAIN (COSTS OFF) SELECT * FROM brin_test_bloom WHERE b = 1;
+ QUERY PLAN
+-----------------------------
+ Seq Scan on brin_test_bloom
+ Filter: (b = 1)
+(2 rows)
+
diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out
index 1b3c146e4c..dca8e9eb34 100644
--- a/src/test/regress/expected/opr_sanity.out
+++ b/src/test/regress/expected/opr_sanity.out
@@ -2034,6 +2034,7 @@ ORDER BY 1, 2, 3;
2742 | 16 | @@
3580 | 1 | <
3580 | 1 | <<
+ 3580 | 1 | =
3580 | 2 | &<
3580 | 2 | <=
3580 | 3 | &&
@@ -2097,7 +2098,7 @@ ORDER BY 1, 2, 3;
4000 | 26 | >>
4000 | 27 | >>=
4000 | 28 | ^@
-(125 rows)
+(126 rows)
-- Check that all opclass search operators have selectivity estimators.
-- This is not absolutely required, but it seems a reasonable thing
diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out
index 555d464f91..c1d19b1e0c 100644
--- a/src/test/regress/expected/psql.out
+++ b/src/test/regress/expected/psql.out
@@ -4929,8 +4929,9 @@ List of access methods
List of operator classes
AM | Input type | Storage type | Operator class | Default?
------+------------+--------------+----------------+----------
+ brin | oid | | oid_bloom_ops | no
brin | oid | | oid_minmax_ops | yes
-(1 row)
+(2 rows)
\dAf spgist
List of operator families
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index 026ea880cd..b49239b1d0 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -75,6 +75,11 @@ test: select_into select_distinct select_distinct_on select_implicit select_havi
# ----------
test: brin gin gist spgist privileges init_privs security_label collate matview lock replica_identity rowsecurity object_address tablesample groupingsets drop_operator password identity generated join_hash
+# ----------
+# Additional BRIN tests
+# ----------
+test: brin_bloom
+
# ----------
# Another group of parallel tests
# ----------
diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule
index 979d926119..abce8e180a 100644
--- a/src/test/regress/serial_schedule
+++ b/src/test/regress/serial_schedule
@@ -107,6 +107,7 @@ test: delete
test: namespace
test: prepared_xacts
test: brin
+test: brin_bloom
test: gin
test: gist
test: spgist
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
new file mode 100644
index 0000000000..abd5a33deb
--- /dev/null
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -0,0 +1,404 @@
+CREATE TABLE brintest_bloom (byteacol bytea,
+ charcol "char",
+ namecol name,
+ int8col bigint,
+ int2col smallint,
+ int4col integer,
+ textcol text,
+ oidcol oid,
+ float4col real,
+ float8col double precision,
+ macaddrcol macaddr,
+ inetcol inet,
+ cidrcol cidr,
+ bpcharcol character,
+ datecol date,
+ timecol time without time zone,
+ timestampcol timestamp without time zone,
+ timestamptzcol timestamp with time zone,
+ intervalcol interval,
+ timetzcol time with time zone,
+ numericcol numeric,
+ uuidcol uuid,
+ lsncol pg_lsn
+) WITH (fillfactor=10);
+
+INSERT INTO brintest_bloom SELECT
+ repeat(stringu1, 8)::bytea,
+ substr(stringu1, 1, 1)::"char",
+ stringu1::name, 142857 * tenthous,
+ thousand,
+ twothousand,
+ repeat(stringu1, 8),
+ unique1::oid,
+ (four + 1.0)/(hundred+1),
+ odd::float8 / (tenthous + 1),
+ format('%s:00:%s:00:%s:00', to_hex(odd), to_hex(even), to_hex(hundred))::macaddr,
+ inet '10.2.3.4/24' + tenthous,
+ cidr '10.2.3/24' + tenthous,
+ substr(stringu1, 1, 1)::bpchar,
+ date '1995-08-15' + tenthous,
+ time '01:20:30' + thousand * interval '18.5 second',
+ timestamp '1942-07-23 03:05:09' + tenthous * interval '36.38 hours',
+ timestamptz '1972-10-10 03:00' + thousand * interval '1 hour',
+ justify_days(justify_hours(tenthous * interval '12 minutes')),
+ timetz '01:30:20+02' + hundred * interval '15 seconds',
+ tenthous::numeric(36,30) * fivethous * even / (hundred + 1),
+ format('%s%s-%s-%s-%s-%s%s%s', to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'))::uuid,
+ format('%s/%s%s', odd, even, tenthous)::pg_lsn
+FROM tenk1 ORDER BY unique2 LIMIT 100;
+
+-- throw in some NULL's and different values
+INSERT INTO brintest_bloom (inetcol, cidrcol) SELECT
+ inet 'fe80::6e40:8ff:fea9:8c46' + tenthous,
+ cidr 'fe80::6e40:8ff:fea9:8c46' + tenthous
+FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
+
+-- test bloom specific index options
+-- ndistinct must be >= -1.0
+CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
+ byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
+);
+-- false_positive_rate must be between 0.001 and 1.0
+CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
+ byteacol bytea_bloom_ops(false_positive_rate = 0.0)
+);
+CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
+ byteacol bytea_bloom_ops(false_positive_rate = 1.01)
+);
+
+CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
+ byteacol bytea_bloom_ops,
+ charcol char_bloom_ops,
+ namecol name_bloom_ops,
+ int8col int8_bloom_ops,
+ int2col int2_bloom_ops,
+ int4col int4_bloom_ops,
+ textcol text_bloom_ops,
+ oidcol oid_bloom_ops,
+ float4col float4_bloom_ops,
+ float8col float8_bloom_ops,
+ macaddrcol macaddr_bloom_ops,
+ inetcol inet_bloom_ops,
+ cidrcol inet_bloom_ops,
+ bpcharcol bpchar_bloom_ops,
+ datecol date_bloom_ops,
+ timecol time_bloom_ops,
+ timestampcol timestamp_bloom_ops,
+ timestamptzcol timestamptz_bloom_ops,
+ intervalcol interval_bloom_ops,
+ timetzcol timetz_bloom_ops,
+ numericcol numeric_bloom_ops,
+ uuidcol uuid_bloom_ops,
+ lsncol pg_lsn_bloom_ops
+) with (pages_per_range = 1);
+
+CREATE TABLE brinopers_bloom (colname name, typ text,
+ op text[], value text[], matches int[],
+ check (cardinality(op) = cardinality(value)),
+ check (cardinality(op) = cardinality(matches)));
+
+INSERT INTO brinopers_bloom VALUES
+ ('byteacol', 'bytea',
+ '{=}',
+ '{BNAAAABNAAAABNAAAABNAAAABNAAAABNAAAABNAAAABNAAAA}',
+ '{1}'),
+ ('charcol', '"char"',
+ '{=}',
+ '{M}',
+ '{6}'),
+ ('namecol', 'name',
+ '{=}',
+ '{MAAAAA}',
+ '{2}'),
+ ('int2col', 'int2',
+ '{=}',
+ '{800}',
+ '{1}'),
+ ('int2col', 'int4',
+ '{=}',
+ '{800}',
+ '{1}'),
+ ('int2col', 'int8',
+ '{=}',
+ '{800}',
+ '{1}'),
+ ('int4col', 'int2',
+ '{=}',
+ '{800}',
+ '{1}'),
+ ('int4col', 'int4',
+ '{=}',
+ '{800}',
+ '{1}'),
+ ('int4col', 'int8',
+ '{=}',
+ '{800}',
+ '{1}'),
+ ('int8col', 'int8',
+ '{=}',
+ '{1257141600}',
+ '{1}'),
+ ('textcol', 'text',
+ '{=}',
+ '{BNAAAABNAAAABNAAAABNAAAABNAAAABNAAAABNAAAABNAAAA}',
+ '{1}'),
+ ('oidcol', 'oid',
+ '{=}',
+ '{8800}',
+ '{1}'),
+ ('float4col', 'float4',
+ '{=}',
+ '{1}',
+ '{4}'),
+ ('float4col', 'float8',
+ '{=}',
+ '{1}',
+ '{4}'),
+ ('float8col', 'float4',
+ '{=}',
+ '{0}',
+ '{1}'),
+ ('float8col', 'float8',
+ '{=}',
+ '{0}',
+ '{1}'),
+ ('macaddrcol', 'macaddr',
+ '{=}',
+ '{2c:00:2d:00:16:00}',
+ '{2}'),
+ ('inetcol', 'inet',
+ '{=}',
+ '{10.2.14.231/24}',
+ '{1}'),
+ ('inetcol', 'cidr',
+ '{=}',
+ '{fe80::6e40:8ff:fea9:8c46}',
+ '{1}'),
+ ('cidrcol', 'inet',
+ '{=}',
+ '{10.2.14/24}',
+ '{2}'),
+ ('cidrcol', 'inet',
+ '{=}',
+ '{fe80::6e40:8ff:fea9:8c46}',
+ '{1}'),
+ ('cidrcol', 'cidr',
+ '{=}',
+ '{10.2.14/24}',
+ '{2}'),
+ ('cidrcol', 'cidr',
+ '{=}',
+ '{fe80::6e40:8ff:fea9:8c46}',
+ '{1}'),
+ ('bpcharcol', 'bpchar',
+ '{=}',
+ '{W}',
+ '{6}'),
+ ('datecol', 'date',
+ '{=}',
+ '{2009-12-01}',
+ '{1}'),
+ ('timecol', 'time',
+ '{=}',
+ '{02:28:57}',
+ '{1}'),
+ ('timestampcol', 'timestamp',
+ '{=}',
+ '{1964-03-24 19:26:45}',
+ '{1}'),
+ ('timestampcol', 'timestamptz',
+ '{=}',
+ '{1964-03-24 19:26:45}',
+ '{1}'),
+ ('timestamptzcol', 'timestamptz',
+ '{=}',
+ '{1972-10-19 09:00:00-07}',
+ '{1}'),
+ ('intervalcol', 'interval',
+ '{=}',
+ '{1 mons 13 days 12:24}',
+ '{1}'),
+ ('timetzcol', 'timetz',
+ '{=}',
+ '{01:35:50+02}',
+ '{2}'),
+ ('numericcol', 'numeric',
+ '{=}',
+ '{2268164.347826086956521739130434782609}',
+ '{1}'),
+ ('uuidcol', 'uuid',
+ '{=}',
+ '{52225222-5222-5222-5222-522252225222}',
+ '{1}'),
+ ('lsncol', 'pg_lsn',
+ '{=, IS, IS NOT}',
+ '{44/455222, NULL, NULL}',
+ '{1, 25, 100}');
+
+DO $x$
+DECLARE
+ r record;
+ r2 record;
+ cond text;
+ idx_ctids tid[];
+ ss_ctids tid[];
+ count int;
+ plan_ok bool;
+ plan_line text;
+BEGIN
+ FOR r IN SELECT colname, oper, typ, value[ordinality], matches[ordinality] FROM brinopers_bloom, unnest(op) WITH ORDINALITY AS oper LOOP
+
+ -- prepare the condition
+ IF r.value IS NULL THEN
+ cond := format('%I %s %L', r.colname, r.oper, r.value);
+ ELSE
+ cond := format('%I %s %L::%s', r.colname, r.oper, r.value, r.typ);
+ END IF;
+
+ -- run the query using the brin index
+ SET enable_seqscan = 0;
+ SET enable_bitmapscan = 1;
+
+ plan_ok := false;
+ FOR plan_line IN EXECUTE format($y$EXPLAIN SELECT array_agg(ctid) FROM brintest_bloom WHERE %s $y$, cond) LOOP
+ IF plan_line LIKE '%Bitmap Heap Scan on brintest_bloom%' THEN
+ plan_ok := true;
+ END IF;
+ END LOOP;
+ IF NOT plan_ok THEN
+ RAISE WARNING 'did not get bitmap indexscan plan for %', r;
+ END IF;
+
+ EXECUTE format($y$SELECT array_agg(ctid) FROM brintest_bloom WHERE %s $y$, cond)
+ INTO idx_ctids;
+
+ -- run the query using a seqscan
+ SET enable_seqscan = 1;
+ SET enable_bitmapscan = 0;
+
+ plan_ok := false;
+ FOR plan_line IN EXECUTE format($y$EXPLAIN SELECT array_agg(ctid) FROM brintest_bloom WHERE %s $y$, cond) LOOP
+ IF plan_line LIKE '%Seq Scan on brintest_bloom%' THEN
+ plan_ok := true;
+ END IF;
+ END LOOP;
+ IF NOT plan_ok THEN
+ RAISE WARNING 'did not get seqscan plan for %', r;
+ END IF;
+
+ EXECUTE format($y$SELECT array_agg(ctid) FROM brintest_bloom WHERE %s $y$, cond)
+ INTO ss_ctids;
+
+ -- make sure both return the same results
+ count := array_length(idx_ctids, 1);
+
+ IF NOT (count = array_length(ss_ctids, 1) AND
+ idx_ctids @> ss_ctids AND
+ idx_ctids <@ ss_ctids) THEN
+ -- report the results of each scan to make the differences obvious
+ RAISE WARNING 'something not right in %: count %', r, count;
+ SET enable_seqscan = 1;
+ SET enable_bitmapscan = 0;
+ FOR r2 IN EXECUTE 'SELECT ' || r.colname || ' FROM brintest_bloom WHERE ' || cond LOOP
+ RAISE NOTICE 'seqscan: %', r2;
+ END LOOP;
+
+ SET enable_seqscan = 0;
+ SET enable_bitmapscan = 1;
+ FOR r2 IN EXECUTE 'SELECT ' || r.colname || ' FROM brintest_bloom WHERE ' || cond LOOP
+ RAISE NOTICE 'bitmapscan: %', r2;
+ END LOOP;
+ END IF;
+
+ -- make sure we found expected number of matches
+ IF count != r.matches THEN RAISE WARNING 'unexpected number of results % for %', count, r; END IF;
+ END LOOP;
+END;
+$x$;
+
+RESET enable_seqscan;
+RESET enable_bitmapscan;
+
+INSERT INTO brintest_bloom SELECT
+ repeat(stringu1, 42)::bytea,
+ substr(stringu1, 1, 1)::"char",
+ stringu1::name, 142857 * tenthous,
+ thousand,
+ twothousand,
+ repeat(stringu1, 42),
+ unique1::oid,
+ (four + 1.0)/(hundred+1),
+ odd::float8 / (tenthous + 1),
+ format('%s:00:%s:00:%s:00', to_hex(odd), to_hex(even), to_hex(hundred))::macaddr,
+ inet '10.2.3.4' + tenthous,
+ cidr '10.2.3/24' + tenthous,
+ substr(stringu1, 1, 1)::bpchar,
+ date '1995-08-15' + tenthous,
+ time '01:20:30' + thousand * interval '18.5 second',
+ timestamp '1942-07-23 03:05:09' + tenthous * interval '36.38 hours',
+ timestamptz '1972-10-10 03:00' + thousand * interval '1 hour',
+ justify_days(justify_hours(tenthous * interval '12 minutes')),
+ timetz '01:30:20' + hundred * interval '15 seconds',
+ tenthous::numeric(36,30) * fivethous * even / (hundred + 1),
+ format('%s%s-%s-%s-%s-%s%s%s', to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'), to_char(tenthous, 'FM0000'))::uuid,
+ format('%s/%s%s', odd, even, tenthous)::pg_lsn
+FROM tenk1 ORDER BY unique2 LIMIT 5 OFFSET 5;
+
+SELECT brin_desummarize_range('brinidx_bloom', 0);
+VACUUM brintest_bloom; -- force a summarization cycle in brinidx
+
+UPDATE brintest_bloom SET int8col = int8col * int4col;
+UPDATE brintest_bloom SET textcol = '' WHERE textcol IS NOT NULL;
+
+-- Tests for brin_summarize_new_values
+SELECT brin_summarize_new_values('brintest_bloom'); -- error, not an index
+SELECT brin_summarize_new_values('tenk1_unique1'); -- error, not a BRIN index
+SELECT brin_summarize_new_values('brinidx_bloom'); -- ok, no change expected
+
+-- Tests for brin_desummarize_range
+SELECT brin_desummarize_range('brinidx_bloom', -1); -- error, invalid range
+SELECT brin_desummarize_range('brinidx_bloom', 0);
+SELECT brin_desummarize_range('brinidx_bloom', 0);
+SELECT brin_desummarize_range('brinidx_bloom', 100000000);
+
+-- Test brin_summarize_range
+CREATE TABLE brin_summarize_bloom (
+ value int
+) WITH (fillfactor=10, autovacuum_enabled=false);
+CREATE INDEX brin_summarize_bloom_idx ON brin_summarize_bloom USING brin (value) WITH (pages_per_range=2);
+-- Fill a few pages
+DO $$
+DECLARE curtid tid;
+BEGIN
+ LOOP
+ INSERT INTO brin_summarize_bloom VALUES (1) RETURNING ctid INTO curtid;
+ EXIT WHEN curtid > tid '(2, 0)';
+ END LOOP;
+END;
+$$;
+
+-- summarize one range
+SELECT brin_summarize_range('brin_summarize_bloom_idx', 0);
+-- nothing: already summarized
+SELECT brin_summarize_range('brin_summarize_bloom_idx', 1);
+-- summarize one range
+SELECT brin_summarize_range('brin_summarize_bloom_idx', 2);
+-- nothing: page doesn't exist in table
+SELECT brin_summarize_range('brin_summarize_bloom_idx', 4294967295);
+-- invalid block number values
+SELECT brin_summarize_range('brin_summarize_bloom_idx', -1);
+SELECT brin_summarize_range('brin_summarize_bloom_idx', 4294967296);
+
+
+-- test brin cost estimates behave sanely based on correlation of values
+CREATE TABLE brin_test_bloom (a INT, b INT);
+INSERT INTO brin_test_bloom SELECT x/100,x%100 FROM generate_series(1,10000) x(x);
+CREATE INDEX brin_test_bloom_a_idx ON brin_test_bloom USING brin (a) WITH (pages_per_range = 2);
+CREATE INDEX brin_test_bloom_b_idx ON brin_test_bloom USING brin (b) WITH (pages_per_range = 2);
+VACUUM ANALYZE brin_test_bloom;
+
+-- Ensure brin index is used when columns are perfectly correlated
+EXPLAIN (COSTS OFF) SELECT * FROM brin_test_bloom WHERE a = 1;
+-- Ensure brin index is not used when values are not correlated
+EXPLAIN (COSTS OFF) SELECT * FROM brin_test_bloom WHERE b = 1;
--
2.25.4
--jsivyprvf2oxfjz3
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
filename="0005-add-special-pg_brin_bloom_summary-data-type-20200807.patch"
^ permalink raw reply [nested|flat] 6+ messages in thread
* Re: [PoC] Federated Authn/z with OAUTHBEARER
@ 2025-04-23 15:39 Christoph Berg <[email protected]>
2025-04-23 16:07 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Jacob Champion <[email protected]>
0 siblings, 1 reply; 6+ messages in thread
From: Christoph Berg @ 2025-04-23 15:39 UTC (permalink / raw)
To: Jacob Champion <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Peter Eisentraut <[email protected]>; Andres Freund <[email protected]>; Tom Lane <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers; Daniel Gustafsson <[email protected]>; Thomas Munro <[email protected]>; Nazir Bilal Yavuz <[email protected]>; Antonin Houska <[email protected]>; Wolfgang Walther <[email protected]>; Devrim Gündüz <[email protected]>
Re: Jacob Champion
> - Per ABI comment upthread, we are back to major-minor versioning for
> the shared library (e.g. libpq-oauth-18-0.so). 0001 adds the macros
> and makefile variables to make this easy, and 0002 is the bulk of the
> change now.
This will cause problems when programs are running while packages are
updated on disk. That program then tries to dlopen 18-0.so when there
is already 18-1.so installed. Relevant when the first oauth connection
is made way after startup.
This is trading one problem for another, but within-a-major ABI
changes should be much rarer than normal minor updates with
applications restarting only later.
Alternatively, there could be a dedicated SONAME for the plugin that
only changes when necessary, but perhaps the simple "18.so" solution
is good enough.
Christoph
^ permalink raw reply [nested|flat] 6+ messages in thread
* Re: [PoC] Federated Authn/z with OAUTHBEARER
2025-04-23 15:39 Re: [PoC] Federated Authn/z with OAUTHBEARER Christoph Berg <[email protected]>
@ 2025-04-23 16:07 ` Jacob Champion <[email protected]>
2025-04-23 16:38 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Christoph Berg <[email protected]>
0 siblings, 1 reply; 6+ messages in thread
From: Jacob Champion @ 2025-04-23 16:07 UTC (permalink / raw)
To: Christoph Berg <[email protected]>; Jacob Champion <[email protected]>; Jelte Fennema-Nio <[email protected]>; Peter Eisentraut <[email protected]>; Andres Freund <[email protected]>; Tom Lane <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers; Daniel Gustafsson <[email protected]>; Thomas Munro <[email protected]>; Nazir Bilal Yavuz <[email protected]>; Antonin Houska <[email protected]>; Wolfgang Walther <[email protected]>; Devrim Gündüz <[email protected]>
On Wed, Apr 23, 2025 at 8:39 AM Christoph Berg <[email protected]> wrote:
> This will cause problems when programs are running while packages are
> updated on disk. That program then tries to dlopen 18-0.so when there
> is already 18-1.so installed. Relevant when the first oauth connection
> is made way after startup.
Ugh, good point. This hazard applies to the previous suggestion of
pkglibdir, too, but in that case it would have been silent...
> This is trading one problem for another, but within-a-major ABI
> changes should be much rarer than normal minor updates with
> applications restarting only later.
But the consequences are much worse for a silent ABI mismatch. Imagine
if libpq-oauth examines the wrong pointer inside PGconn for a
security-critical check.
> Alternatively, there could be a dedicated SONAME for the plugin that
> only changes when necessary, but perhaps the simple "18.so" solution
> is good enough.
I don't think SONAME helps us, does it? We're not using it in dlopen().
We could all agree to bump the second number in the filename whenever
there's an internal ABI change. That works from a technical
perspective, but it's hard to test and enforce and... just not forget.
Or, I may still be able to thread the needle with a fuller lookup
table, and remove the dependency on libpq-int.h; it's just not going
to be incredibly pretty. Thinking...
Thanks so much for your continued review!
--Jacob
^ permalink raw reply [nested|flat] 6+ messages in thread
* Re: [PoC] Federated Authn/z with OAUTHBEARER
2025-04-23 15:39 Re: [PoC] Federated Authn/z with OAUTHBEARER Christoph Berg <[email protected]>
2025-04-23 16:07 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Jacob Champion <[email protected]>
@ 2025-04-23 16:38 ` Christoph Berg <[email protected]>
2025-04-23 17:46 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Jacob Champion <[email protected]>
0 siblings, 1 reply; 6+ messages in thread
From: Christoph Berg @ 2025-04-23 16:38 UTC (permalink / raw)
To: Jacob Champion <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Peter Eisentraut <[email protected]>; Andres Freund <[email protected]>; Tom Lane <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers; Daniel Gustafsson <[email protected]>; Thomas Munro <[email protected]>; Nazir Bilal Yavuz <[email protected]>; Antonin Houska <[email protected]>; Wolfgang Walther <[email protected]>; Devrim Gündüz <[email protected]>
Re: Jacob Champion
> But the consequences are much worse for a silent ABI mismatch. Imagine
> if libpq-oauth examines the wrong pointer inside PGconn for a
> security-critical check.
True.
> > Alternatively, there could be a dedicated SONAME for the plugin that
> > only changes when necessary, but perhaps the simple "18.so" solution
> > is good enough.
>
> I don't think SONAME helps us, does it? We're not using it in dlopen().
That was paraphrasing, with SONAME I meant "library file name that
changes when the ABI changes".
> We could all agree to bump the second number in the filename whenever
> there's an internal ABI change. That works from a technical
> perspective, but it's hard to test and enforce and... just not forget.
It's hopefully not harder than checking ABI compatibility of any other
libpq change, just a different number. If that number is in the
meson.build in the same directory, people should be able to connect
the dots.
Btw, if we have that number, we might as well drop the MAJOR part as
well... apt.pg.o is always shipping the latest libpq5, so major libpq
upgrades while apps are running are going to happen. (But this is just
once a year and much less problematic than minor upgrades and I'm not
going to complain if MAJOR is kept.)
> Or, I may still be able to thread the needle with a fuller lookup
> table, and remove the dependency on libpq-int.h; it's just not going
> to be incredibly pretty. Thinking...
Don't overdesign it...
Christoph
^ permalink raw reply [nested|flat] 6+ messages in thread
* Re: [PoC] Federated Authn/z with OAUTHBEARER
2025-04-23 15:39 Re: [PoC] Federated Authn/z with OAUTHBEARER Christoph Berg <[email protected]>
2025-04-23 16:07 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Jacob Champion <[email protected]>
2025-04-23 16:38 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Christoph Berg <[email protected]>
@ 2025-04-23 17:46 ` Jacob Champion <[email protected]>
2025-04-29 00:10 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Jacob Champion <[email protected]>
0 siblings, 1 reply; 6+ messages in thread
From: Jacob Champion @ 2025-04-23 17:46 UTC (permalink / raw)
To: Christoph Berg <[email protected]>; Jacob Champion <[email protected]>; Jelte Fennema-Nio <[email protected]>; Peter Eisentraut <[email protected]>; Andres Freund <[email protected]>; Tom Lane <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers; Daniel Gustafsson <[email protected]>; Thomas Munro <[email protected]>; Nazir Bilal Yavuz <[email protected]>; Antonin Houska <[email protected]>; Wolfgang Walther <[email protected]>; Devrim Gündüz <[email protected]>
On Wed, Apr 23, 2025 at 9:38 AM Christoph Berg <[email protected]> wrote:
> > We could all agree to bump the second number in the filename whenever
> > there's an internal ABI change. That works from a technical
> > perspective, but it's hard to test and enforce and... just not forget.
>
> It's hopefully not harder than checking ABI compatibility of any other
> libpq change, just a different number. If that number is in the
> meson.build in the same directory, people should be able to connect
> the dots.
I think it is harder, simply because no one has to do it today, and
that change would sign them up to do it, forever, adding to the
backport checklist. It's one thing if there's a bunch of committers
who pile into the thread right now saying "yes, that's okay", but I
don't really feel comfortable making that decision for them right this
instant.
If we had robust ABI compatibility checks as part of the farm [1], I
think we could do that. Doesn't feel like an 18 thing, though.
> Btw, if we have that number, we might as well drop the MAJOR part as
> well... apt.pg.o is always shipping the latest libpq5, so major libpq
> upgrades while apps are running are going to happen. (But this is just
> once a year and much less problematic than minor upgrades and I'm not
> going to complain if MAJOR is kept.)
I don't want to introduce another testing matrix dimension if I can
avoid it. ("I have this bug where libpq.so.5.18 is using
libpq-oauth.so from PG20 and I had no idea it was doing that and the
problem went away when I restarted and...")
And the intent is for this to be temporary until we have a user-facing
API. If this is the solution we go with, I think it'd wise to prepare
for a -19 version of libpq-oauth, but I'm going to try my best to get
custom modules in ASAP. People are going to be annoyed that v1 of the
feature doesn't let them swap the flow for our utilities. Ideally they
only have to deal with that for a single major release.
Also: since the libpq-oauth-18 and libpq-oauth-19 packages can be
installed side-by-side safely, isn't the upgrade hazard significantly
diminished? (If a user uninstalls the previous libpq-oauth version
while they're still running that version of libpq in memory, _and_
they've somehow never used OAuth until right that instant... it's easy
enough for them to undo their mistake while the application is still
running.)
> > Or, I may still be able to thread the needle with a fuller lookup
> > table, and remove the dependency on libpq-int.h; it's just not going
> > to be incredibly pretty. Thinking...
>
> Don't overdesign it...
Oh, I agree... but my "minimal" ABI designs have all had corner cases
so far. I may need to just bite the bullet.
Are there any readers who feel like an internal ABI version for
`struct pg_conn`, bumped during breaking backports, would be
acceptable? (More definitively: are there any readers who would veto
that?) You're still signing up for delayed errors in the long-lived
client case, so it's not a magic bullet, but the breakage is easy to
see and it's not a crash. The client application "just" has to restart
after a libpq upgrade.
Thanks,
--Jacob
[1] https://www.postgresql.org/message-id/B142EE8A-5D38-48B9-A4BB-82D69A854B55%40justatheory.com
^ permalink raw reply [nested|flat] 6+ messages in thread
* Re: [PoC] Federated Authn/z with OAUTHBEARER
2025-04-23 15:39 Re: [PoC] Federated Authn/z with OAUTHBEARER Christoph Berg <[email protected]>
2025-04-23 16:07 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Jacob Champion <[email protected]>
2025-04-23 16:38 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Christoph Berg <[email protected]>
2025-04-23 17:46 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Jacob Champion <[email protected]>
@ 2025-04-29 00:10 ` Jacob Champion <[email protected]>
0 siblings, 0 replies; 6+ messages in thread
From: Jacob Champion @ 2025-04-29 00:10 UTC (permalink / raw)
To: Christoph Berg <[email protected]>; Jacob Champion <[email protected]>; Jelte Fennema-Nio <[email protected]>; Peter Eisentraut <[email protected]>; Andres Freund <[email protected]>; Tom Lane <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers; Daniel Gustafsson <[email protected]>; Thomas Munro <[email protected]>; Nazir Bilal Yavuz <[email protected]>; Antonin Houska <[email protected]>; Wolfgang Walther <[email protected]>; Devrim Gündüz <[email protected]>
On Wed, Apr 23, 2025 at 10:46 AM Jacob Champion
<[email protected]> wrote:
> Are there any readers who feel like an internal ABI version for
> `struct pg_conn`, bumped during breaking backports, would be
> acceptable? (More definitively: are there any readers who would veto
> that?)
To keep things moving: I assume this is unacceptable. So v10 redirects
every access to a PGconn struct member through a shim, similarly to
how conn->errorMessage was translated in v9. This adds plenty of new
boilerplate, but not a whole lot of complexity. To try to keep us
honest, libpq-int.h has been removed from the libpq-oauth includes.
This will now handle in-place minor version upgrades that swap pg_conn
internals around, so I've gone back to -MAJOR versioning alone.
fe_oauth_state is still exported; it now has an ABI warning above it.
(I figure that's easier to draw a line around during backports,
compared to everything in PGconn. We can still break things there
during major version upgrades.)
Thanks,
--Jacob
1: 5f87f11b18e < -: ----------- Add minor-version counterpart to (PG_)MAJORVERSION
2: 9e37fd7c217 ! 1: e86e93f7ac8 oauth: Move the builtin flow into a separate module
@@ Commit message
The default flow relies on some libpq internals. Some of these can be
safely duplicated (such as the SIGPIPE handlers), but others need to be
- shared between libpq and libpq-oauth for thread-safety. To avoid exporting
- these internals to all libpq clients forever, these dependencies are
- instead injected from the libpq side via an initialization function.
- This also lets libpq communicate the offset of conn->errorMessage to
- libpq-oauth, so that we can function without crashing if the module on
- the search path came from a different build of Postgres.
+ shared between libpq and libpq-oauth for thread-safety. To avoid
+ exporting these internals to all libpq clients forever, these
+ dependencies are instead injected from the libpq side via an
+ initialization function. This also lets libpq communicate the offsets of
+ PGconn struct members to libpq-oauth, so that we can function without
+ crashing if the module on the search path came from a different build of
+ Postgres. (A minor-version upgrade could swap the libpq-oauth module out
+ from under a long-running libpq client before it does its first load of
+ the OAuth flow.)
This ABI is considered "private". The module has no SONAME or version
- symlinks, and it's named libpq-oauth-<major>-<minor>.so to avoid mixing
- and matching across Postgres versions, in case internal struct order
- needs to change. (Future improvements may promote this "OAuth flow
- plugin" to a first-class concept, at which point we would need a public
- API to replace this anyway.)
+ symlinks, and it's named libpq-oauth-<major>.so to avoid mixing and
+ matching across Postgres versions. (Future improvements may promote this
+ "OAuth flow plugin" to a first-class concept, at which point we would
+ need a public API to replace this anyway.)
Additionally, NLS support for error messages in b3f0be788a was
incomplete, because the new error macros weren't being scanned by
@@ src/interfaces/libpq-oauth/Makefile (new)
+
+# This is an internal module; we don't want an SONAME and therefore do not set
+# SO_MAJOR_VERSION.
-+NAME = pq-oauth-$(MAJORVERSION)-$(MINORVERSION)
++NAME = pq-oauth-$(MAJORVERSION)
+
+# Force the name "libpq-oauth" for both the static and shared libraries. The
+# staticlib doesn't need version information in its name.
@@ src/interfaces/libpq-oauth/README (new)
+= Load-Time ABI =
+
+This module ABI is an internal implementation detail, so it's subject to change
-+across releases; the name of the module (libpq-oauth-MAJOR-MINOR) reflects this.
++across major releases; the name of the module (libpq-oauth-MAJOR) reflects this.
+The module exports the following symbols:
+
+- PostgresPollingStatusType pg_fe_run_oauth_flow(PGconn *conn);
@@ src/interfaces/libpq-oauth/README (new)
+
+- void libpq_oauth_init(pgthreadlock_t threadlock,
+ libpq_gettext_func gettext_impl,
-+ conn_errorMessage_func errmsg_impl);
++ conn_errorMessage_func errmsg_impl,
++ conn_oauth_client_id_func clientid_impl,
++ conn_oauth_client_secret_func clientsecret_impl,
++ conn_oauth_discovery_uri_func discoveryuri_impl,
++ conn_oauth_issuer_id_func issuerid_impl,
++ conn_oauth_scope_func scope_impl,
++ conn_sasl_state_func saslstate_impl,
++ set_conn_altsock_func setaltsock_impl,
++ set_conn_oauth_token_func settoken_impl);
+
+At the moment, pg_fe_run_oauth_flow() relies on libpq's pg_g_threadlock and
+libpq_gettext(), which must be injected by libpq using this initialization
+function before the flow is run.
+
-+It also relies on libpq to expose conn->errorMessage, via the errmsg_impl. This
-+is done to decouple the module ABI from the offset of errorMessage, which can
-+change positions depending on configure-time options. This way we can safely
-+search the standard dlopen() paths (e.g. RPATH, LD_LIBRARY_PATH, the SO cache)
-+for an implementation module to use, even if that module wasn't compiled at the
-+same time as libpq.
++It also relies on access to several members of the PGconn struct. Not only can
++these change positions across minor versions, but the offsets aren't necessarily
++stable within a single minor release (conn->errorMessage, for instance, can
++change offsets depending on configure-time options). Therefore the necessary
++accessors (named conn_*) and mutators (set_conn_*) are injected here. With this
++approach, we can safely search the standard dlopen() paths (e.g. RPATH,
++LD_LIBRARY_PATH, the SO cache) for an implementation module to use, even if that
++module wasn't compiled at the same time as libpq -- which becomes especially
++important during "live upgrade" situations where a running libpq application has
++the libpq-oauth module updated out from under it before it's first loaded from
++disk.
+
+= Static Build =
+
+The static library libpq.a does not perform any dynamic loading. If the builtin
+flow is enabled, the application is expected to link against libpq-oauth.a
-+directly to provide the necessary symbols.
++directly to provide the necessary symbols. (libpq.a and libpq-oauth.a must be
++part of the same build. Unlike the dynamic module, there are no translation
++shims provided.)
## src/interfaces/libpq-oauth/exports.txt (new) ##
@@
@@ src/interfaces/libpq-oauth/meson.build (new)
+
+# This is an internal module; we don't want an SONAME and therefore do not set
+# SO_MAJOR_VERSION.
-+libpq_oauth_name = 'libpq-oauth-@0@-@1@'.format(pg_version_major, pg_version_minor)
++libpq_oauth_name = 'libpq-oauth-@0@'.format(pg_version_major)
+
+libpq_oauth_so = shared_module(libpq_oauth_name,
+ libpq_oauth_sources + libpq_oauth_so_sources,
@@ src/interfaces/libpq/fe-auth-oauth-curl.c => src/interfaces/libpq-oauth/oauth-cu
-#include <unistd.h>
#include "common/jsonapi.h"
- #include "fe-auth.h"
+-#include "fe-auth.h"
#include "fe-auth-oauth.h"
-#include "libpq-int.h"
#include "mb/pg_wchar.h"
+#include "oauth-curl.h"
++
+#ifdef USE_DYNAMIC_OAUTH
++
++/*
++ * The module build is decoupled from libpq-int.h, to try to avoid inadvertent
++ * ABI breaks during minor version bumps. Replacements for the missing internals
++ * are provided by oauth-utils.
++ */
+#include "oauth-utils.h"
++
++#else /* !USE_DYNAMIC_OAUTH */
++
++/*
++ * Static builds may rely on PGconn offsets directly. Keep these aligned with
++ * the bank of callbacks in oauth-utils.h.
++ */
++#include "libpq-int.h"
++
++#define conn_errorMessage(CONN) (&CONN->errorMessage)
++#define conn_oauth_client_id(CONN) (CONN->oauth_client_id)
++#define conn_oauth_client_secret(CONN) (CONN->oauth_client_secret)
++#define conn_oauth_discovery_uri(CONN) (CONN->oauth_discovery_uri)
++#define conn_oauth_issuer_id(CONN) (CONN->oauth_issuer_id)
++#define conn_oauth_scope(CONN) (CONN->oauth_scope)
++#define conn_sasl_state(CONN) (CONN->sasl_state)
++
++#define set_conn_altsock(CONN, VAL) do { CONN->altsock = VAL; } while (0)
++#define set_conn_oauth_token(CONN, VAL) do { CONN->oauth_token = VAL; } while (0)
++
++#endif /* !USE_DYNAMIC_OAUTH */
++
++/* One final guardrail against accidental inclusion... */
++#if defined(USE_DYNAMIC_OAUTH) && defined(LIBPQ_INT_H)
++#error do not rely on libpq-int.h in libpq-oauth.so
+#endif
/*
* It's generally prudent to set a maximum response size to buffer in memory,
+@@ src/interfaces/libpq-oauth/oauth-curl.c: free_async_ctx(PGconn *conn, struct async_ctx *actx)
+ void
+ pg_fe_cleanup_oauth_flow(PGconn *conn)
+ {
+- fe_oauth_state *state = conn->sasl_state;
++ fe_oauth_state *state = conn_sasl_state(conn);
+
+ if (state->async_ctx)
+ {
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_cleanup_oauth_flow(PGconn *conn)
+ state->async_ctx = NULL;
+ }
+
+- conn->altsock = PGINVALID_SOCKET;
++ set_conn_altsock(conn, PGINVALID_SOCKET);
+ }
+
+ /*
@@ src/interfaces/libpq-oauth/oauth-curl.c: parse_access_token(struct async_ctx *actx, struct token *tok)
static bool
setup_multiplexer(struct async_ctx *actx)
@@ src/interfaces/libpq-oauth/oauth-curl.c: timer_expired(struct async_ctx *actx)
}
/*
+@@ src/interfaces/libpq-oauth/oauth-curl.c: static bool
+ check_issuer(struct async_ctx *actx, PGconn *conn)
+ {
+ const struct provider *provider = &actx->provider;
++ const char *oauth_issuer_id = conn_oauth_issuer_id(conn);
+
+- Assert(conn->oauth_issuer_id); /* ensured by setup_oauth_parameters() */
++ Assert(oauth_issuer_id); /* ensured by setup_oauth_parameters() */
+ Assert(provider->issuer); /* ensured by parse_provider() */
+
+ /*---
+@@ src/interfaces/libpq-oauth/oauth-curl.c: check_issuer(struct async_ctx *actx, PGconn *conn)
+ * sent to. This comparison MUST use simple string comparison as defined
+ * in Section 6.2.1 of [RFC3986].
+ */
+- if (strcmp(conn->oauth_issuer_id, provider->issuer) != 0)
++ if (strcmp(oauth_issuer_id, provider->issuer) != 0)
+ {
+ actx_error(actx,
+ "the issuer identifier (%s) does not match oauth_issuer (%s)",
+- provider->issuer, conn->oauth_issuer_id);
++ provider->issuer, oauth_issuer_id);
+ return false;
+ }
+
+@@ src/interfaces/libpq-oauth/oauth-curl.c: check_for_device_flow(struct async_ctx *actx)
+ static bool
+ add_client_identification(struct async_ctx *actx, PQExpBuffer reqbody, PGconn *conn)
+ {
++ const char *oauth_client_id = conn_oauth_client_id(conn);
++ const char *oauth_client_secret = conn_oauth_client_secret(conn);
++
+ bool success = false;
+ char *username = NULL;
+ char *password = NULL;
+
+- if (conn->oauth_client_secret) /* Zero-length secrets are permitted! */
++ if (oauth_client_secret) /* Zero-length secrets are permitted! */
+ {
+ /*----
+ * Use HTTP Basic auth to send the client_id and secret. Per RFC 6749,
+@@ src/interfaces/libpq-oauth/oauth-curl.c: add_client_identification(struct async_ctx *actx, PQExpBuffer reqbody, PGconn *c
+ * would it be redundant, but some providers in the wild (e.g. Okta)
+ * refuse to accept it.
+ */
+- username = urlencode(conn->oauth_client_id);
+- password = urlencode(conn->oauth_client_secret);
++ username = urlencode(oauth_client_id);
++ password = urlencode(oauth_client_secret);
+
+ if (!username || !password)
+ {
+@@ src/interfaces/libpq-oauth/oauth-curl.c: add_client_identification(struct async_ctx *actx, PQExpBuffer reqbody, PGconn *c
+ * If we're not otherwise authenticating, client_id is REQUIRED in the
+ * request body.
+ */
+- build_urlencoded(reqbody, "client_id", conn->oauth_client_id);
++ build_urlencoded(reqbody, "client_id", oauth_client_id);
+
+ CHECK_SETOPT(actx, CURLOPT_HTTPAUTH, CURLAUTH_NONE, goto cleanup);
+ actx->used_basic_auth = false;
+@@ src/interfaces/libpq-oauth/oauth-curl.c: cleanup:
+ static bool
+ start_device_authz(struct async_ctx *actx, PGconn *conn)
+ {
++ const char *oauth_scope = conn_oauth_scope(conn);
+ const char *device_authz_uri = actx->provider.device_authorization_endpoint;
+ PQExpBuffer work_buffer = &actx->work_data;
+
+- Assert(conn->oauth_client_id); /* ensured by setup_oauth_parameters() */
++ Assert(conn_oauth_client_id(conn)); /* ensured by setup_oauth_parameters() */
+ Assert(device_authz_uri); /* ensured by check_for_device_flow() */
+
+ /* Construct our request body. */
+ resetPQExpBuffer(work_buffer);
+- if (conn->oauth_scope && conn->oauth_scope[0])
+- build_urlencoded(work_buffer, "scope", conn->oauth_scope);
++ if (oauth_scope && oauth_scope[0])
++ build_urlencoded(work_buffer, "scope", oauth_scope);
+
+ if (!add_client_identification(actx, work_buffer, conn))
+ return false;
+@@ src/interfaces/libpq-oauth/oauth-curl.c: start_token_request(struct async_ctx *actx, PGconn *conn)
+ const char *device_code = actx->authz.device_code;
+ PQExpBuffer work_buffer = &actx->work_data;
+
+- Assert(conn->oauth_client_id); /* ensured by setup_oauth_parameters() */
++ Assert(conn_oauth_client_id(conn)); /* ensured by setup_oauth_parameters() */
+ Assert(token_uri); /* ensured by parse_provider() */
+ Assert(device_code); /* ensured by parse_device_authz() */
+
@@ src/interfaces/libpq-oauth/oauth-curl.c: prompt_user(struct async_ctx *actx, PGconn *conn)
.verification_uri_complete = actx->authz.verification_uri_complete,
.expires_in = actx->authz.expires_in,
@@ src/interfaces/libpq-oauth/oauth-curl.c: prompt_user(struct async_ctx *actx, PGc
if (!res)
{
-@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+@@ src/interfaces/libpq-oauth/oauth-curl.c: done:
+ static PostgresPollingStatusType
+ pg_fe_run_oauth_flow_impl(PGconn *conn)
{
- fe_oauth_state *state = conn->sasl_state;
+- fe_oauth_state *state = conn->sasl_state;
++ fe_oauth_state *state = conn_sasl_state(conn);
struct async_ctx *actx;
++ char *oauth_token = NULL;
+ PQExpBuffer errbuf;
if (!initialize_curl(conn))
return PGRES_POLLING_FAILED;
@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ do
+ {
+ /* By default, the multiplexer is the altsock. Reassign as desired. */
+- conn->altsock = actx->mux;
++ set_conn_altsock(conn, actx->mux);
+
+ switch (actx->step)
+ {
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ */
+ if (!timer_expired(actx))
+ {
+- conn->altsock = actx->timerfd;
++ set_conn_altsock(conn, actx->timerfd);
+ return PGRES_POLLING_READING;
+ }
- error_return:
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ {
+ case OAUTH_STEP_INIT:
+ actx->errctx = "failed to fetch OpenID discovery document";
+- if (!start_discovery(actx, conn->oauth_discovery_uri))
++ if (!start_discovery(actx, conn_oauth_discovery_uri(conn)))
+ goto error_return;
-+ /*
-+ * For the dynamic module build, we can't safely rely on the offset of
-+ * conn->errorMessage, since it depends on build options like USE_SSL et
-+ * al. libpq gives us a translator function instead.
-+ */
-+#ifdef USE_DYNAMIC_OAUTH
+ actx->step = OAUTH_STEP_DISCOVERY;
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ break;
+
+ case OAUTH_STEP_TOKEN_REQUEST:
+- if (!handle_token_response(actx, &conn->oauth_token))
++ if (!handle_token_response(actx, &oauth_token))
+ goto error_return;
+
++ /*
++ * Hook any oauth_token into the PGconn immediately so that
++ * the allocation isn't lost in case of an error.
++ */
++ set_conn_oauth_token(conn, oauth_token);
++
+ if (!actx->user_prompted)
+ {
+ /*
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ actx->user_prompted = true;
+ }
+
+- if (conn->oauth_token)
++ if (oauth_token)
+ break; /* done! */
+
+ /*
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ * the client wait directly on the timerfd rather than the
+ * multiplexer.
+ */
+- conn->altsock = actx->timerfd;
++ set_conn_altsock(conn, actx->timerfd);
+
+ actx->step = OAUTH_STEP_WAIT_INTERVAL;
+ actx->running = 1;
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ * point, actx->running will be set. But there are some corner cases
+ * where we can immediately loop back around; see start_request().
+ */
+- } while (!conn->oauth_token && !actx->running);
++ } while (!oauth_token && !actx->running);
+
+ /* If we've stored a token, we're done. Otherwise come back later. */
+- return conn->oauth_token ? PGRES_POLLING_OK : PGRES_POLLING_READING;
++ return oauth_token ? PGRES_POLLING_OK : PGRES_POLLING_READING;
+
+ error_return:
+ errbuf = conn_errorMessage(conn);
-+#else
-+ errbuf = &conn->errorMessage;
-+#endif
-+
+
/*
* Assemble the three parts of our error: context, body, and detail. See
* also the documentation for struct async_ctx.
@@ src/interfaces/libpq-oauth/oauth-utils.c (new)
+
+#include <signal.h>
+
-+#include "libpq-int.h"
+#include "oauth-utils.h"
+
+#ifndef USE_DYNAMIC_OAUTH
+#error oauth-utils.c is not supported in static builds
+#endif
+
-+static libpq_gettext_func libpq_gettext_impl;
++#ifdef LIBPQ_INT_H
++#error do not rely on libpq-int.h in libpq-oauth
++#endif
++
++/*
++ * Function pointers set by libpq_oauth_init().
++ */
+
+pgthreadlock_t pg_g_threadlock;
++static libpq_gettext_func libpq_gettext_impl;
++
+conn_errorMessage_func conn_errorMessage;
++conn_oauth_client_id_func conn_oauth_client_id;
++conn_oauth_client_secret_func conn_oauth_client_secret;
++conn_oauth_discovery_uri_func conn_oauth_discovery_uri;
++conn_oauth_issuer_id_func conn_oauth_issuer_id;
++conn_oauth_scope_func conn_oauth_scope;
++conn_sasl_state_func conn_sasl_state;
++
++set_conn_altsock_func set_conn_altsock;
++set_conn_oauth_token_func set_conn_oauth_token;
+
+/*-
+ * Initializes libpq-oauth by setting necessary callbacks.
@@ src/interfaces/libpq-oauth/oauth-utils.c (new)
+ *
+ * - libpq_gettext: translates error messages using libpq's message domain
+ *
-+ * - conn->errorMessage: holds translated errors for the connection. This is
-+ * handled through a translation shim, which avoids either depending on the
-+ * offset of the errorMessage in PGconn, or needing to export the variadic
-+ * libpq_append_conn_error().
++ * The implementation also needs access to several members of the PGconn struct,
++ * which are not guaranteed to stay in place across minor versions. Accessors
++ * (named conn_*) and mutators (named set_conn_*) are injected here.
+ */
+void
+libpq_oauth_init(pgthreadlock_t threadlock_impl,
+ libpq_gettext_func gettext_impl,
-+ conn_errorMessage_func errmsg_impl)
++ conn_errorMessage_func errmsg_impl,
++ conn_oauth_client_id_func clientid_impl,
++ conn_oauth_client_secret_func clientsecret_impl,
++ conn_oauth_discovery_uri_func discoveryuri_impl,
++ conn_oauth_issuer_id_func issuerid_impl,
++ conn_oauth_scope_func scope_impl,
++ conn_sasl_state_func saslstate_impl,
++ set_conn_altsock_func setaltsock_impl,
++ set_conn_oauth_token_func settoken_impl)
+{
+ pg_g_threadlock = threadlock_impl;
+ libpq_gettext_impl = gettext_impl;
+ conn_errorMessage = errmsg_impl;
++ conn_oauth_client_id = clientid_impl;
++ conn_oauth_client_secret = clientsecret_impl;
++ conn_oauth_discovery_uri = discoveryuri_impl;
++ conn_oauth_issuer_id = issuerid_impl;
++ conn_oauth_scope = scope_impl;
++ conn_sasl_state = saslstate_impl;
++ set_conn_altsock = setaltsock_impl;
++ set_conn_oauth_token = settoken_impl;
+}
+
+/*
@@ src/interfaces/libpq-oauth/oauth-utils.h (new)
+#ifndef OAUTH_UTILS_H
+#define OAUTH_UTILS_H
+
++#include "fe-auth-oauth.h"
+#include "libpq-fe.h"
+#include "pqexpbuffer.h"
+
++/*
++ * A bank of callbacks to safely access members of PGconn, which are all passed
++ * to libpq_oauth_init() by libpq.
++ *
++ * Keep these aligned with the definitions in fe-auth-oauth.c as well as the
++ * static declarations in oauth-curl.c.
++ */
++#define DECLARE_GETTER(TYPE, MEMBER) \
++ typedef TYPE (*conn_ ## MEMBER ## _func) (PGconn *conn); \
++ extern conn_ ## MEMBER ## _func conn_ ## MEMBER;
++
++#define DECLARE_SETTER(TYPE, MEMBER) \
++ typedef void (*set_conn_ ## MEMBER ## _func) (PGconn *conn, TYPE val); \
++ extern set_conn_ ## MEMBER ## _func set_conn_ ## MEMBER;
++
++DECLARE_GETTER(PQExpBuffer, errorMessage);
++DECLARE_GETTER(char *, oauth_client_id);
++DECLARE_GETTER(char *, oauth_client_secret);
++DECLARE_GETTER(char *, oauth_discovery_uri);
++DECLARE_GETTER(char *, oauth_issuer_id);
++DECLARE_GETTER(char *, oauth_scope);
++DECLARE_GETTER(fe_oauth_state *, sasl_state);
++
++DECLARE_SETTER(pgsocket, altsock);
++DECLARE_SETTER(char *, oauth_token);
++
++#undef DECLARE_GETTER
++#undef DECLARE_SETTER
++
+typedef char *(*libpq_gettext_func) (const char *msgid);
-+typedef PQExpBuffer (*conn_errorMessage_func) (PGconn *conn);
+
+/* Initializes libpq-oauth. */
+extern PGDLLEXPORT void libpq_oauth_init(pgthreadlock_t threadlock,
+ libpq_gettext_func gettext_impl,
-+ conn_errorMessage_func errmsg_impl);
++ conn_errorMessage_func errmsg_impl,
++ conn_oauth_client_id_func clientid_impl,
++ conn_oauth_client_secret_func clientsecret_impl,
++ conn_oauth_discovery_uri_func discoveryuri_impl,
++ conn_oauth_issuer_id_func issuerid_impl,
++ conn_oauth_scope_func scope_impl,
++ conn_sasl_state_func saslstate_impl,
++ set_conn_altsock_func setaltsock_impl,
++ set_conn_oauth_token_func settoken_impl);
+
-+/* Callback to safely obtain conn->errorMessage from a PGconn. */
-+extern conn_errorMessage_func conn_errorMessage;
++/*
++ * Duplicated APIs, copied from libpq (primarily libpq-int.h, which we cannot
++ * depend on here).
++ */
++
++typedef enum
++{
++ PG_BOOL_UNKNOWN = 0, /* Currently unknown */
++ PG_BOOL_YES, /* Yes (true) */
++ PG_BOOL_NO /* No (false) */
++} PGTernaryBool;
+
-+/* Duplicated APIs, copied from libpq. */
+extern void libpq_append_conn_error(PGconn *conn, const char *fmt,...) pg_attribute_printf(2, 3);
+extern bool oauth_unsafe_debugging_enabled(void);
+extern int pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending);
+extern void pq_reset_sigpipe(sigset_t *osigset, bool sigpipe_pending, bool got_epipe);
+
++#ifdef ENABLE_NLS
++extern char *libpq_gettext(const char *msgid) pg_attribute_format_arg(1);
++#else
++#define libpq_gettext(x) (x)
++#endif
++
++extern pgthreadlock_t pg_g_threadlock;
++
++#define pglock_thread() pg_g_threadlock(true)
++#define pgunlock_thread() pg_g_threadlock(false)
++
+#endif /* OAUTH_UTILS_H */
## src/interfaces/libpq/Makefile ##
@@ src/interfaces/libpq/fe-auth-oauth.c: cleanup_user_oauth_flow(PGconn *conn)
+ */
+
+typedef char *(*libpq_gettext_func) (const char *msgid);
-+typedef PQExpBuffer (*conn_errorMessage_func) (PGconn *conn);
+
+/*
-+ * This shim is injected into libpq-oauth so that it doesn't depend on the
-+ * offset of conn->errorMessage.
-+ *
-+ * TODO: look into exporting libpq_append_conn_error or a comparable API from
-+ * libpq, instead.
++ * Define accessor/mutator shims to inject into libpq-oauth, so that it doesn't
++ * depend on the offsets within PGconn. (These have changed during minor version
++ * updates in the past.)
+ */
-+static PQExpBuffer
-+conn_errorMessage(PGconn *conn)
-+{
-+ return &conn->errorMessage;
-+}
++
++#define DEFINE_GETTER(TYPE, MEMBER) \
++ typedef TYPE (*conn_ ## MEMBER ## _func) (PGconn *conn); \
++ static TYPE conn_ ## MEMBER(PGconn *conn) { return conn->MEMBER; }
++
++/* Like DEFINE_GETTER, but returns a pointer to the member. */
++#define DEFINE_GETTER_P(TYPE, MEMBER) \
++ typedef TYPE (*conn_ ## MEMBER ## _func) (PGconn *conn); \
++ static TYPE conn_ ## MEMBER(PGconn *conn) { return &conn->MEMBER; }
++
++#define DEFINE_SETTER(TYPE, MEMBER) \
++ typedef void (*set_conn_ ## MEMBER ## _func) (PGconn *conn, TYPE val); \
++ static void set_conn_ ## MEMBER(PGconn *conn, TYPE val) { conn->MEMBER = val; }
++
++DEFINE_GETTER_P(PQExpBuffer, errorMessage);
++DEFINE_GETTER(char *, oauth_client_id);
++DEFINE_GETTER(char *, oauth_client_secret);
++DEFINE_GETTER(char *, oauth_discovery_uri);
++DEFINE_GETTER(char *, oauth_issuer_id);
++DEFINE_GETTER(char *, oauth_scope);
++DEFINE_GETTER(fe_oauth_state *, sasl_state);
++
++DEFINE_SETTER(pgsocket, altsock);
++DEFINE_SETTER(char *, oauth_token);
+
+/*
+ * Loads the libpq-oauth plugin via dlopen(), initializes it, and plugs its
@@ src/interfaces/libpq/fe-auth-oauth.c: cleanup_user_oauth_flow(PGconn *conn)
+
+ void (*init) (pgthreadlock_t threadlock,
+ libpq_gettext_func gettext_impl,
-+ conn_errorMessage_func errmsg_impl);
++ conn_errorMessage_func errmsg_impl,
++ conn_oauth_client_id_func clientid_impl,
++ conn_oauth_client_secret_func clientsecret_impl,
++ conn_oauth_discovery_uri_func discoveryuri_impl,
++ conn_oauth_issuer_id_func issuerid_impl,
++ conn_oauth_scope_func scope_impl,
++ conn_sasl_state_func saslstate_impl,
++ set_conn_altsock_func setaltsock_impl,
++ set_conn_oauth_token_func settoken_impl);
+ PostgresPollingStatusType (*flow) (PGconn *conn);
+ void (*cleanup) (PGconn *conn);
+
@@ src/interfaces/libpq/fe-auth-oauth.c: cleanup_user_oauth_flow(PGconn *conn)
+ */
+ const char *const module_name =
+#if defined(__darwin__)
-+ LIBDIR "/libpq-oauth-" PG_MAJORVERSION "-" PG_MINORVERSION DLSUFFIX;
++ LIBDIR "/libpq-oauth-" PG_MAJORVERSION DLSUFFIX;
+#else
-+ "libpq-oauth-" PG_MAJORVERSION "-" PG_MINORVERSION DLSUFFIX;
++ "libpq-oauth-" PG_MAJORVERSION DLSUFFIX;
+#endif
+
+ state->builtin_flow = dlopen(module_name, RTLD_NOW | RTLD_LOCAL);
@@ src/interfaces/libpq/fe-auth-oauth.c: cleanup_user_oauth_flow(PGconn *conn)
+#else
+ NULL,
+#endif
-+ conn_errorMessage);
++ conn_errorMessage,
++ conn_oauth_client_id,
++ conn_oauth_client_secret,
++ conn_oauth_discovery_uri,
++ conn_oauth_issuer_id,
++ conn_oauth_scope,
++ conn_sasl_state,
++ set_conn_altsock,
++ set_conn_oauth_token);
+
+ initialized = true;
+ }
@@ src/interfaces/libpq/fe-auth-oauth.c: setup_token_request(PGconn *conn, fe_oauth
return true;
## src/interfaces/libpq/fe-auth-oauth.h ##
-@@ src/interfaces/libpq/fe-auth-oauth.h: typedef struct
+@@
+ #ifndef FE_AUTH_OAUTH_H
+ #define FE_AUTH_OAUTH_H
+
++#include "fe-auth-sasl.h"
+ #include "libpq-fe.h"
+-#include "libpq-int.h"
+
+
+ enum fe_oauth_step
+@@ src/interfaces/libpq/fe-auth-oauth.h: enum fe_oauth_step
+ FE_OAUTH_SERVER_ERROR,
+ };
+
++/*
++ * This struct is exported to the libpq-oauth module. If changes are needed
++ * during backports to stable branches, please keep ABI compatibility (no
++ * changes to existing members, add new members at the end, etc.).
++ */
+ typedef struct
+ {
+ enum fe_oauth_step step;
PGconn *conn;
void *async_ctx;
Attachments:
[text/plain] since-v9.diff.txt (28.4K, ../../CAOYmi+=UsSnLM4K+hYkhrezpQROQ5jr=72feAkQ0Te8GJf3Fbg@mail.gmail.com/2-since-v9.diff.txt)
download | inline:
1: 5f87f11b18e < -: ----------- Add minor-version counterpart to (PG_)MAJORVERSION
2: 9e37fd7c217 ! 1: e86e93f7ac8 oauth: Move the builtin flow into a separate module
@@ Commit message
The default flow relies on some libpq internals. Some of these can be
safely duplicated (such as the SIGPIPE handlers), but others need to be
- shared between libpq and libpq-oauth for thread-safety. To avoid exporting
- these internals to all libpq clients forever, these dependencies are
- instead injected from the libpq side via an initialization function.
- This also lets libpq communicate the offset of conn->errorMessage to
- libpq-oauth, so that we can function without crashing if the module on
- the search path came from a different build of Postgres.
+ shared between libpq and libpq-oauth for thread-safety. To avoid
+ exporting these internals to all libpq clients forever, these
+ dependencies are instead injected from the libpq side via an
+ initialization function. This also lets libpq communicate the offsets of
+ PGconn struct members to libpq-oauth, so that we can function without
+ crashing if the module on the search path came from a different build of
+ Postgres. (A minor-version upgrade could swap the libpq-oauth module out
+ from under a long-running libpq client before it does its first load of
+ the OAuth flow.)
This ABI is considered "private". The module has no SONAME or version
- symlinks, and it's named libpq-oauth-<major>-<minor>.so to avoid mixing
- and matching across Postgres versions, in case internal struct order
- needs to change. (Future improvements may promote this "OAuth flow
- plugin" to a first-class concept, at which point we would need a public
- API to replace this anyway.)
+ symlinks, and it's named libpq-oauth-<major>.so to avoid mixing and
+ matching across Postgres versions. (Future improvements may promote this
+ "OAuth flow plugin" to a first-class concept, at which point we would
+ need a public API to replace this anyway.)
Additionally, NLS support for error messages in b3f0be788a was
incomplete, because the new error macros weren't being scanned by
@@ src/interfaces/libpq-oauth/Makefile (new)
+
+# This is an internal module; we don't want an SONAME and therefore do not set
+# SO_MAJOR_VERSION.
-+NAME = pq-oauth-$(MAJORVERSION)-$(MINORVERSION)
++NAME = pq-oauth-$(MAJORVERSION)
+
+# Force the name "libpq-oauth" for both the static and shared libraries. The
+# staticlib doesn't need version information in its name.
@@ src/interfaces/libpq-oauth/README (new)
+= Load-Time ABI =
+
+This module ABI is an internal implementation detail, so it's subject to change
-+across releases; the name of the module (libpq-oauth-MAJOR-MINOR) reflects this.
++across major releases; the name of the module (libpq-oauth-MAJOR) reflects this.
+The module exports the following symbols:
+
+- PostgresPollingStatusType pg_fe_run_oauth_flow(PGconn *conn);
@@ src/interfaces/libpq-oauth/README (new)
+
+- void libpq_oauth_init(pgthreadlock_t threadlock,
+ libpq_gettext_func gettext_impl,
-+ conn_errorMessage_func errmsg_impl);
++ conn_errorMessage_func errmsg_impl,
++ conn_oauth_client_id_func clientid_impl,
++ conn_oauth_client_secret_func clientsecret_impl,
++ conn_oauth_discovery_uri_func discoveryuri_impl,
++ conn_oauth_issuer_id_func issuerid_impl,
++ conn_oauth_scope_func scope_impl,
++ conn_sasl_state_func saslstate_impl,
++ set_conn_altsock_func setaltsock_impl,
++ set_conn_oauth_token_func settoken_impl);
+
+At the moment, pg_fe_run_oauth_flow() relies on libpq's pg_g_threadlock and
+libpq_gettext(), which must be injected by libpq using this initialization
+function before the flow is run.
+
-+It also relies on libpq to expose conn->errorMessage, via the errmsg_impl. This
-+is done to decouple the module ABI from the offset of errorMessage, which can
-+change positions depending on configure-time options. This way we can safely
-+search the standard dlopen() paths (e.g. RPATH, LD_LIBRARY_PATH, the SO cache)
-+for an implementation module to use, even if that module wasn't compiled at the
-+same time as libpq.
++It also relies on access to several members of the PGconn struct. Not only can
++these change positions across minor versions, but the offsets aren't necessarily
++stable within a single minor release (conn->errorMessage, for instance, can
++change offsets depending on configure-time options). Therefore the necessary
++accessors (named conn_*) and mutators (set_conn_*) are injected here. With this
++approach, we can safely search the standard dlopen() paths (e.g. RPATH,
++LD_LIBRARY_PATH, the SO cache) for an implementation module to use, even if that
++module wasn't compiled at the same time as libpq -- which becomes especially
++important during "live upgrade" situations where a running libpq application has
++the libpq-oauth module updated out from under it before it's first loaded from
++disk.
+
+= Static Build =
+
+The static library libpq.a does not perform any dynamic loading. If the builtin
+flow is enabled, the application is expected to link against libpq-oauth.a
-+directly to provide the necessary symbols.
++directly to provide the necessary symbols. (libpq.a and libpq-oauth.a must be
++part of the same build. Unlike the dynamic module, there are no translation
++shims provided.)
## src/interfaces/libpq-oauth/exports.txt (new) ##
@@
@@ src/interfaces/libpq-oauth/meson.build (new)
+
+# This is an internal module; we don't want an SONAME and therefore do not set
+# SO_MAJOR_VERSION.
-+libpq_oauth_name = 'libpq-oauth-@0@-@1@'.format(pg_version_major, pg_version_minor)
++libpq_oauth_name = 'libpq-oauth-@0@'.format(pg_version_major)
+
+libpq_oauth_so = shared_module(libpq_oauth_name,
+ libpq_oauth_sources + libpq_oauth_so_sources,
@@ src/interfaces/libpq/fe-auth-oauth-curl.c => src/interfaces/libpq-oauth/oauth-cu
-#include <unistd.h>
#include "common/jsonapi.h"
- #include "fe-auth.h"
+-#include "fe-auth.h"
#include "fe-auth-oauth.h"
-#include "libpq-int.h"
#include "mb/pg_wchar.h"
+#include "oauth-curl.h"
++
+#ifdef USE_DYNAMIC_OAUTH
++
++/*
++ * The module build is decoupled from libpq-int.h, to try to avoid inadvertent
++ * ABI breaks during minor version bumps. Replacements for the missing internals
++ * are provided by oauth-utils.
++ */
+#include "oauth-utils.h"
++
++#else /* !USE_DYNAMIC_OAUTH */
++
++/*
++ * Static builds may rely on PGconn offsets directly. Keep these aligned with
++ * the bank of callbacks in oauth-utils.h.
++ */
++#include "libpq-int.h"
++
++#define conn_errorMessage(CONN) (&CONN->errorMessage)
++#define conn_oauth_client_id(CONN) (CONN->oauth_client_id)
++#define conn_oauth_client_secret(CONN) (CONN->oauth_client_secret)
++#define conn_oauth_discovery_uri(CONN) (CONN->oauth_discovery_uri)
++#define conn_oauth_issuer_id(CONN) (CONN->oauth_issuer_id)
++#define conn_oauth_scope(CONN) (CONN->oauth_scope)
++#define conn_sasl_state(CONN) (CONN->sasl_state)
++
++#define set_conn_altsock(CONN, VAL) do { CONN->altsock = VAL; } while (0)
++#define set_conn_oauth_token(CONN, VAL) do { CONN->oauth_token = VAL; } while (0)
++
++#endif /* !USE_DYNAMIC_OAUTH */
++
++/* One final guardrail against accidental inclusion... */
++#if defined(USE_DYNAMIC_OAUTH) && defined(LIBPQ_INT_H)
++#error do not rely on libpq-int.h in libpq-oauth.so
+#endif
/*
* It's generally prudent to set a maximum response size to buffer in memory,
+@@ src/interfaces/libpq-oauth/oauth-curl.c: free_async_ctx(PGconn *conn, struct async_ctx *actx)
+ void
+ pg_fe_cleanup_oauth_flow(PGconn *conn)
+ {
+- fe_oauth_state *state = conn->sasl_state;
++ fe_oauth_state *state = conn_sasl_state(conn);
+
+ if (state->async_ctx)
+ {
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_cleanup_oauth_flow(PGconn *conn)
+ state->async_ctx = NULL;
+ }
+
+- conn->altsock = PGINVALID_SOCKET;
++ set_conn_altsock(conn, PGINVALID_SOCKET);
+ }
+
+ /*
@@ src/interfaces/libpq-oauth/oauth-curl.c: parse_access_token(struct async_ctx *actx, struct token *tok)
static bool
setup_multiplexer(struct async_ctx *actx)
@@ src/interfaces/libpq-oauth/oauth-curl.c: timer_expired(struct async_ctx *actx)
}
/*
+@@ src/interfaces/libpq-oauth/oauth-curl.c: static bool
+ check_issuer(struct async_ctx *actx, PGconn *conn)
+ {
+ const struct provider *provider = &actx->provider;
++ const char *oauth_issuer_id = conn_oauth_issuer_id(conn);
+
+- Assert(conn->oauth_issuer_id); /* ensured by setup_oauth_parameters() */
++ Assert(oauth_issuer_id); /* ensured by setup_oauth_parameters() */
+ Assert(provider->issuer); /* ensured by parse_provider() */
+
+ /*---
+@@ src/interfaces/libpq-oauth/oauth-curl.c: check_issuer(struct async_ctx *actx, PGconn *conn)
+ * sent to. This comparison MUST use simple string comparison as defined
+ * in Section 6.2.1 of [RFC3986].
+ */
+- if (strcmp(conn->oauth_issuer_id, provider->issuer) != 0)
++ if (strcmp(oauth_issuer_id, provider->issuer) != 0)
+ {
+ actx_error(actx,
+ "the issuer identifier (%s) does not match oauth_issuer (%s)",
+- provider->issuer, conn->oauth_issuer_id);
++ provider->issuer, oauth_issuer_id);
+ return false;
+ }
+
+@@ src/interfaces/libpq-oauth/oauth-curl.c: check_for_device_flow(struct async_ctx *actx)
+ static bool
+ add_client_identification(struct async_ctx *actx, PQExpBuffer reqbody, PGconn *conn)
+ {
++ const char *oauth_client_id = conn_oauth_client_id(conn);
++ const char *oauth_client_secret = conn_oauth_client_secret(conn);
++
+ bool success = false;
+ char *username = NULL;
+ char *password = NULL;
+
+- if (conn->oauth_client_secret) /* Zero-length secrets are permitted! */
++ if (oauth_client_secret) /* Zero-length secrets are permitted! */
+ {
+ /*----
+ * Use HTTP Basic auth to send the client_id and secret. Per RFC 6749,
+@@ src/interfaces/libpq-oauth/oauth-curl.c: add_client_identification(struct async_ctx *actx, PQExpBuffer reqbody, PGconn *c
+ * would it be redundant, but some providers in the wild (e.g. Okta)
+ * refuse to accept it.
+ */
+- username = urlencode(conn->oauth_client_id);
+- password = urlencode(conn->oauth_client_secret);
++ username = urlencode(oauth_client_id);
++ password = urlencode(oauth_client_secret);
+
+ if (!username || !password)
+ {
+@@ src/interfaces/libpq-oauth/oauth-curl.c: add_client_identification(struct async_ctx *actx, PQExpBuffer reqbody, PGconn *c
+ * If we're not otherwise authenticating, client_id is REQUIRED in the
+ * request body.
+ */
+- build_urlencoded(reqbody, "client_id", conn->oauth_client_id);
++ build_urlencoded(reqbody, "client_id", oauth_client_id);
+
+ CHECK_SETOPT(actx, CURLOPT_HTTPAUTH, CURLAUTH_NONE, goto cleanup);
+ actx->used_basic_auth = false;
+@@ src/interfaces/libpq-oauth/oauth-curl.c: cleanup:
+ static bool
+ start_device_authz(struct async_ctx *actx, PGconn *conn)
+ {
++ const char *oauth_scope = conn_oauth_scope(conn);
+ const char *device_authz_uri = actx->provider.device_authorization_endpoint;
+ PQExpBuffer work_buffer = &actx->work_data;
+
+- Assert(conn->oauth_client_id); /* ensured by setup_oauth_parameters() */
++ Assert(conn_oauth_client_id(conn)); /* ensured by setup_oauth_parameters() */
+ Assert(device_authz_uri); /* ensured by check_for_device_flow() */
+
+ /* Construct our request body. */
+ resetPQExpBuffer(work_buffer);
+- if (conn->oauth_scope && conn->oauth_scope[0])
+- build_urlencoded(work_buffer, "scope", conn->oauth_scope);
++ if (oauth_scope && oauth_scope[0])
++ build_urlencoded(work_buffer, "scope", oauth_scope);
+
+ if (!add_client_identification(actx, work_buffer, conn))
+ return false;
+@@ src/interfaces/libpq-oauth/oauth-curl.c: start_token_request(struct async_ctx *actx, PGconn *conn)
+ const char *device_code = actx->authz.device_code;
+ PQExpBuffer work_buffer = &actx->work_data;
+
+- Assert(conn->oauth_client_id); /* ensured by setup_oauth_parameters() */
++ Assert(conn_oauth_client_id(conn)); /* ensured by setup_oauth_parameters() */
+ Assert(token_uri); /* ensured by parse_provider() */
+ Assert(device_code); /* ensured by parse_device_authz() */
+
@@ src/interfaces/libpq-oauth/oauth-curl.c: prompt_user(struct async_ctx *actx, PGconn *conn)
.verification_uri_complete = actx->authz.verification_uri_complete,
.expires_in = actx->authz.expires_in,
@@ src/interfaces/libpq-oauth/oauth-curl.c: prompt_user(struct async_ctx *actx, PGc
if (!res)
{
-@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+@@ src/interfaces/libpq-oauth/oauth-curl.c: done:
+ static PostgresPollingStatusType
+ pg_fe_run_oauth_flow_impl(PGconn *conn)
{
- fe_oauth_state *state = conn->sasl_state;
+- fe_oauth_state *state = conn->sasl_state;
++ fe_oauth_state *state = conn_sasl_state(conn);
struct async_ctx *actx;
++ char *oauth_token = NULL;
+ PQExpBuffer errbuf;
if (!initialize_curl(conn))
return PGRES_POLLING_FAILED;
@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ do
+ {
+ /* By default, the multiplexer is the altsock. Reassign as desired. */
+- conn->altsock = actx->mux;
++ set_conn_altsock(conn, actx->mux);
+
+ switch (actx->step)
+ {
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ */
+ if (!timer_expired(actx))
+ {
+- conn->altsock = actx->timerfd;
++ set_conn_altsock(conn, actx->timerfd);
+ return PGRES_POLLING_READING;
+ }
- error_return:
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ {
+ case OAUTH_STEP_INIT:
+ actx->errctx = "failed to fetch OpenID discovery document";
+- if (!start_discovery(actx, conn->oauth_discovery_uri))
++ if (!start_discovery(actx, conn_oauth_discovery_uri(conn)))
+ goto error_return;
-+ /*
-+ * For the dynamic module build, we can't safely rely on the offset of
-+ * conn->errorMessage, since it depends on build options like USE_SSL et
-+ * al. libpq gives us a translator function instead.
-+ */
-+#ifdef USE_DYNAMIC_OAUTH
+ actx->step = OAUTH_STEP_DISCOVERY;
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ break;
+
+ case OAUTH_STEP_TOKEN_REQUEST:
+- if (!handle_token_response(actx, &conn->oauth_token))
++ if (!handle_token_response(actx, &oauth_token))
+ goto error_return;
+
++ /*
++ * Hook any oauth_token into the PGconn immediately so that
++ * the allocation isn't lost in case of an error.
++ */
++ set_conn_oauth_token(conn, oauth_token);
++
+ if (!actx->user_prompted)
+ {
+ /*
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ actx->user_prompted = true;
+ }
+
+- if (conn->oauth_token)
++ if (oauth_token)
+ break; /* done! */
+
+ /*
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ * the client wait directly on the timerfd rather than the
+ * multiplexer.
+ */
+- conn->altsock = actx->timerfd;
++ set_conn_altsock(conn, actx->timerfd);
+
+ actx->step = OAUTH_STEP_WAIT_INTERVAL;
+ actx->running = 1;
+@@ src/interfaces/libpq-oauth/oauth-curl.c: pg_fe_run_oauth_flow_impl(PGconn *conn)
+ * point, actx->running will be set. But there are some corner cases
+ * where we can immediately loop back around; see start_request().
+ */
+- } while (!conn->oauth_token && !actx->running);
++ } while (!oauth_token && !actx->running);
+
+ /* If we've stored a token, we're done. Otherwise come back later. */
+- return conn->oauth_token ? PGRES_POLLING_OK : PGRES_POLLING_READING;
++ return oauth_token ? PGRES_POLLING_OK : PGRES_POLLING_READING;
+
+ error_return:
+ errbuf = conn_errorMessage(conn);
-+#else
-+ errbuf = &conn->errorMessage;
-+#endif
-+
+
/*
* Assemble the three parts of our error: context, body, and detail. See
* also the documentation for struct async_ctx.
@@ src/interfaces/libpq-oauth/oauth-utils.c (new)
+
+#include <signal.h>
+
-+#include "libpq-int.h"
+#include "oauth-utils.h"
+
+#ifndef USE_DYNAMIC_OAUTH
+#error oauth-utils.c is not supported in static builds
+#endif
+
-+static libpq_gettext_func libpq_gettext_impl;
++#ifdef LIBPQ_INT_H
++#error do not rely on libpq-int.h in libpq-oauth
++#endif
++
++/*
++ * Function pointers set by libpq_oauth_init().
++ */
+
+pgthreadlock_t pg_g_threadlock;
++static libpq_gettext_func libpq_gettext_impl;
++
+conn_errorMessage_func conn_errorMessage;
++conn_oauth_client_id_func conn_oauth_client_id;
++conn_oauth_client_secret_func conn_oauth_client_secret;
++conn_oauth_discovery_uri_func conn_oauth_discovery_uri;
++conn_oauth_issuer_id_func conn_oauth_issuer_id;
++conn_oauth_scope_func conn_oauth_scope;
++conn_sasl_state_func conn_sasl_state;
++
++set_conn_altsock_func set_conn_altsock;
++set_conn_oauth_token_func set_conn_oauth_token;
+
+/*-
+ * Initializes libpq-oauth by setting necessary callbacks.
@@ src/interfaces/libpq-oauth/oauth-utils.c (new)
+ *
+ * - libpq_gettext: translates error messages using libpq's message domain
+ *
-+ * - conn->errorMessage: holds translated errors for the connection. This is
-+ * handled through a translation shim, which avoids either depending on the
-+ * offset of the errorMessage in PGconn, or needing to export the variadic
-+ * libpq_append_conn_error().
++ * The implementation also needs access to several members of the PGconn struct,
++ * which are not guaranteed to stay in place across minor versions. Accessors
++ * (named conn_*) and mutators (named set_conn_*) are injected here.
+ */
+void
+libpq_oauth_init(pgthreadlock_t threadlock_impl,
+ libpq_gettext_func gettext_impl,
-+ conn_errorMessage_func errmsg_impl)
++ conn_errorMessage_func errmsg_impl,
++ conn_oauth_client_id_func clientid_impl,
++ conn_oauth_client_secret_func clientsecret_impl,
++ conn_oauth_discovery_uri_func discoveryuri_impl,
++ conn_oauth_issuer_id_func issuerid_impl,
++ conn_oauth_scope_func scope_impl,
++ conn_sasl_state_func saslstate_impl,
++ set_conn_altsock_func setaltsock_impl,
++ set_conn_oauth_token_func settoken_impl)
+{
+ pg_g_threadlock = threadlock_impl;
+ libpq_gettext_impl = gettext_impl;
+ conn_errorMessage = errmsg_impl;
++ conn_oauth_client_id = clientid_impl;
++ conn_oauth_client_secret = clientsecret_impl;
++ conn_oauth_discovery_uri = discoveryuri_impl;
++ conn_oauth_issuer_id = issuerid_impl;
++ conn_oauth_scope = scope_impl;
++ conn_sasl_state = saslstate_impl;
++ set_conn_altsock = setaltsock_impl;
++ set_conn_oauth_token = settoken_impl;
+}
+
+/*
@@ src/interfaces/libpq-oauth/oauth-utils.h (new)
+#ifndef OAUTH_UTILS_H
+#define OAUTH_UTILS_H
+
++#include "fe-auth-oauth.h"
+#include "libpq-fe.h"
+#include "pqexpbuffer.h"
+
++/*
++ * A bank of callbacks to safely access members of PGconn, which are all passed
++ * to libpq_oauth_init() by libpq.
++ *
++ * Keep these aligned with the definitions in fe-auth-oauth.c as well as the
++ * static declarations in oauth-curl.c.
++ */
++#define DECLARE_GETTER(TYPE, MEMBER) \
++ typedef TYPE (*conn_ ## MEMBER ## _func) (PGconn *conn); \
++ extern conn_ ## MEMBER ## _func conn_ ## MEMBER;
++
++#define DECLARE_SETTER(TYPE, MEMBER) \
++ typedef void (*set_conn_ ## MEMBER ## _func) (PGconn *conn, TYPE val); \
++ extern set_conn_ ## MEMBER ## _func set_conn_ ## MEMBER;
++
++DECLARE_GETTER(PQExpBuffer, errorMessage);
++DECLARE_GETTER(char *, oauth_client_id);
++DECLARE_GETTER(char *, oauth_client_secret);
++DECLARE_GETTER(char *, oauth_discovery_uri);
++DECLARE_GETTER(char *, oauth_issuer_id);
++DECLARE_GETTER(char *, oauth_scope);
++DECLARE_GETTER(fe_oauth_state *, sasl_state);
++
++DECLARE_SETTER(pgsocket, altsock);
++DECLARE_SETTER(char *, oauth_token);
++
++#undef DECLARE_GETTER
++#undef DECLARE_SETTER
++
+typedef char *(*libpq_gettext_func) (const char *msgid);
-+typedef PQExpBuffer (*conn_errorMessage_func) (PGconn *conn);
+
+/* Initializes libpq-oauth. */
+extern PGDLLEXPORT void libpq_oauth_init(pgthreadlock_t threadlock,
+ libpq_gettext_func gettext_impl,
-+ conn_errorMessage_func errmsg_impl);
++ conn_errorMessage_func errmsg_impl,
++ conn_oauth_client_id_func clientid_impl,
++ conn_oauth_client_secret_func clientsecret_impl,
++ conn_oauth_discovery_uri_func discoveryuri_impl,
++ conn_oauth_issuer_id_func issuerid_impl,
++ conn_oauth_scope_func scope_impl,
++ conn_sasl_state_func saslstate_impl,
++ set_conn_altsock_func setaltsock_impl,
++ set_conn_oauth_token_func settoken_impl);
+
-+/* Callback to safely obtain conn->errorMessage from a PGconn. */
-+extern conn_errorMessage_func conn_errorMessage;
++/*
++ * Duplicated APIs, copied from libpq (primarily libpq-int.h, which we cannot
++ * depend on here).
++ */
++
++typedef enum
++{
++ PG_BOOL_UNKNOWN = 0, /* Currently unknown */
++ PG_BOOL_YES, /* Yes (true) */
++ PG_BOOL_NO /* No (false) */
++} PGTernaryBool;
+
-+/* Duplicated APIs, copied from libpq. */
+extern void libpq_append_conn_error(PGconn *conn, const char *fmt,...) pg_attribute_printf(2, 3);
+extern bool oauth_unsafe_debugging_enabled(void);
+extern int pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending);
+extern void pq_reset_sigpipe(sigset_t *osigset, bool sigpipe_pending, bool got_epipe);
+
++#ifdef ENABLE_NLS
++extern char *libpq_gettext(const char *msgid) pg_attribute_format_arg(1);
++#else
++#define libpq_gettext(x) (x)
++#endif
++
++extern pgthreadlock_t pg_g_threadlock;
++
++#define pglock_thread() pg_g_threadlock(true)
++#define pgunlock_thread() pg_g_threadlock(false)
++
+#endif /* OAUTH_UTILS_H */
## src/interfaces/libpq/Makefile ##
@@ src/interfaces/libpq/fe-auth-oauth.c: cleanup_user_oauth_flow(PGconn *conn)
+ */
+
+typedef char *(*libpq_gettext_func) (const char *msgid);
-+typedef PQExpBuffer (*conn_errorMessage_func) (PGconn *conn);
+
+/*
-+ * This shim is injected into libpq-oauth so that it doesn't depend on the
-+ * offset of conn->errorMessage.
-+ *
-+ * TODO: look into exporting libpq_append_conn_error or a comparable API from
-+ * libpq, instead.
++ * Define accessor/mutator shims to inject into libpq-oauth, so that it doesn't
++ * depend on the offsets within PGconn. (These have changed during minor version
++ * updates in the past.)
+ */
-+static PQExpBuffer
-+conn_errorMessage(PGconn *conn)
-+{
-+ return &conn->errorMessage;
-+}
++
++#define DEFINE_GETTER(TYPE, MEMBER) \
++ typedef TYPE (*conn_ ## MEMBER ## _func) (PGconn *conn); \
++ static TYPE conn_ ## MEMBER(PGconn *conn) { return conn->MEMBER; }
++
++/* Like DEFINE_GETTER, but returns a pointer to the member. */
++#define DEFINE_GETTER_P(TYPE, MEMBER) \
++ typedef TYPE (*conn_ ## MEMBER ## _func) (PGconn *conn); \
++ static TYPE conn_ ## MEMBER(PGconn *conn) { return &conn->MEMBER; }
++
++#define DEFINE_SETTER(TYPE, MEMBER) \
++ typedef void (*set_conn_ ## MEMBER ## _func) (PGconn *conn, TYPE val); \
++ static void set_conn_ ## MEMBER(PGconn *conn, TYPE val) { conn->MEMBER = val; }
++
++DEFINE_GETTER_P(PQExpBuffer, errorMessage);
++DEFINE_GETTER(char *, oauth_client_id);
++DEFINE_GETTER(char *, oauth_client_secret);
++DEFINE_GETTER(char *, oauth_discovery_uri);
++DEFINE_GETTER(char *, oauth_issuer_id);
++DEFINE_GETTER(char *, oauth_scope);
++DEFINE_GETTER(fe_oauth_state *, sasl_state);
++
++DEFINE_SETTER(pgsocket, altsock);
++DEFINE_SETTER(char *, oauth_token);
+
+/*
+ * Loads the libpq-oauth plugin via dlopen(), initializes it, and plugs its
@@ src/interfaces/libpq/fe-auth-oauth.c: cleanup_user_oauth_flow(PGconn *conn)
+
+ void (*init) (pgthreadlock_t threadlock,
+ libpq_gettext_func gettext_impl,
-+ conn_errorMessage_func errmsg_impl);
++ conn_errorMessage_func errmsg_impl,
++ conn_oauth_client_id_func clientid_impl,
++ conn_oauth_client_secret_func clientsecret_impl,
++ conn_oauth_discovery_uri_func discoveryuri_impl,
++ conn_oauth_issuer_id_func issuerid_impl,
++ conn_oauth_scope_func scope_impl,
++ conn_sasl_state_func saslstate_impl,
++ set_conn_altsock_func setaltsock_impl,
++ set_conn_oauth_token_func settoken_impl);
+ PostgresPollingStatusType (*flow) (PGconn *conn);
+ void (*cleanup) (PGconn *conn);
+
@@ src/interfaces/libpq/fe-auth-oauth.c: cleanup_user_oauth_flow(PGconn *conn)
+ */
+ const char *const module_name =
+#if defined(__darwin__)
-+ LIBDIR "/libpq-oauth-" PG_MAJORVERSION "-" PG_MINORVERSION DLSUFFIX;
++ LIBDIR "/libpq-oauth-" PG_MAJORVERSION DLSUFFIX;
+#else
-+ "libpq-oauth-" PG_MAJORVERSION "-" PG_MINORVERSION DLSUFFIX;
++ "libpq-oauth-" PG_MAJORVERSION DLSUFFIX;
+#endif
+
+ state->builtin_flow = dlopen(module_name, RTLD_NOW | RTLD_LOCAL);
@@ src/interfaces/libpq/fe-auth-oauth.c: cleanup_user_oauth_flow(PGconn *conn)
+#else
+ NULL,
+#endif
-+ conn_errorMessage);
++ conn_errorMessage,
++ conn_oauth_client_id,
++ conn_oauth_client_secret,
++ conn_oauth_discovery_uri,
++ conn_oauth_issuer_id,
++ conn_oauth_scope,
++ conn_sasl_state,
++ set_conn_altsock,
++ set_conn_oauth_token);
+
+ initialized = true;
+ }
@@ src/interfaces/libpq/fe-auth-oauth.c: setup_token_request(PGconn *conn, fe_oauth
return true;
## src/interfaces/libpq/fe-auth-oauth.h ##
-@@ src/interfaces/libpq/fe-auth-oauth.h: typedef struct
+@@
+ #ifndef FE_AUTH_OAUTH_H
+ #define FE_AUTH_OAUTH_H
+
++#include "fe-auth-sasl.h"
+ #include "libpq-fe.h"
+-#include "libpq-int.h"
+
+
+ enum fe_oauth_step
+@@ src/interfaces/libpq/fe-auth-oauth.h: enum fe_oauth_step
+ FE_OAUTH_SERVER_ERROR,
+ };
+
++/*
++ * This struct is exported to the libpq-oauth module. If changes are needed
++ * during backports to stable branches, please keep ABI compatibility (no
++ * changes to existing members, add new members at the end, etc.).
++ */
+ typedef struct
+ {
+ enum fe_oauth_step step;
PGconn *conn;
void *async_ctx;
[application/x-patch] v10-0001-oauth-Move-the-builtin-flow-into-a-separate-modu.patch (69.6K, ../../CAOYmi+=UsSnLM4K+hYkhrezpQROQ5jr=72feAkQ0Te8GJf3Fbg@mail.gmail.com/3-v10-0001-oauth-Move-the-builtin-flow-into-a-separate-modu.patch)
download | inline diff:
From e86e93f7ac8e0ee746b95d804b8367a6ea4c9d30 Mon Sep 17 00:00:00 2001
From: Jacob Champion <[email protected]>
Date: Wed, 26 Mar 2025 10:55:28 -0700
Subject: [PATCH v10] oauth: Move the builtin flow into a separate module
The additional packaging footprint of the OAuth Curl dependency, as well
as the existence of libcurl in the address space even if OAuth isn't
ever used by a client, has raised some concerns. Split off this
dependency into a separate loadable module called libpq-oauth.
When configured using --with-libcurl, libpq.so searches for this new
module via dlopen(). End users may choose not to install the libpq-oauth
module, in which case the default flow is disabled.
For static applications using libpq.a, the libpq-oauth staticlib is a
mandatory link-time dependency for --with-libcurl builds. libpq.pc has
been updated accordingly.
The default flow relies on some libpq internals. Some of these can be
safely duplicated (such as the SIGPIPE handlers), but others need to be
shared between libpq and libpq-oauth for thread-safety. To avoid
exporting these internals to all libpq clients forever, these
dependencies are instead injected from the libpq side via an
initialization function. This also lets libpq communicate the offsets of
PGconn struct members to libpq-oauth, so that we can function without
crashing if the module on the search path came from a different build of
Postgres. (A minor-version upgrade could swap the libpq-oauth module out
from under a long-running libpq client before it does its first load of
the OAuth flow.)
This ABI is considered "private". The module has no SONAME or version
symlinks, and it's named libpq-oauth-<major>.so to avoid mixing and
matching across Postgres versions. (Future improvements may promote this
"OAuth flow plugin" to a first-class concept, at which point we would
need a public API to replace this anyway.)
Additionally, NLS support for error messages in b3f0be788a was
incomplete, because the new error macros weren't being scanned by
xgettext. Fix that now.
Per request from Tom Lane and Bruce Momjian. Based on an initial patch
by Daniel Gustafsson, who also contributed docs changes. The "bare"
dlopen() concept came from Thomas Munro. Many many people reviewed the
design and implementation; thank you!
Co-authored-by: Daniel Gustafsson <[email protected]>
Reviewed-by: Andres Freund <[email protected]>
Reviewed-by: Christoph Berg <[email protected]>
Reviewed-by: Jelte Fennema-Nio <[email protected]>
Reviewed-by: Peter Eisentraut <[email protected]>
Reviewed-by: Wolfgang Walther <[email protected]>
Discussion: https://postgr.es/m/641687.1742360249%40sss.pgh.pa.us
---
config/programs.m4 | 17 +-
configure | 50 +++-
configure.ac | 26 +-
doc/src/sgml/installation.sgml | 8 +
doc/src/sgml/libpq.sgml | 30 ++-
meson.build | 32 ++-
src/Makefile.global.in | 3 +
src/interfaces/Makefile | 12 +
src/interfaces/libpq-oauth/Makefile | 83 +++++++
src/interfaces/libpq-oauth/README | 58 +++++
src/interfaces/libpq-oauth/exports.txt | 4 +
src/interfaces/libpq-oauth/meson.build | 45 ++++
.../oauth-curl.c} | 180 ++++++++------
src/interfaces/libpq-oauth/oauth-curl.h | 24 ++
src/interfaces/libpq-oauth/oauth-utils.c | 233 ++++++++++++++++++
src/interfaces/libpq-oauth/oauth-utils.h | 94 +++++++
src/interfaces/libpq/Makefile | 36 ++-
src/interfaces/libpq/exports.txt | 1 +
src/interfaces/libpq/fe-auth-oauth.c | 229 ++++++++++++++++-
src/interfaces/libpq/fe-auth-oauth.h | 12 +-
src/interfaces/libpq/meson.build | 25 +-
src/interfaces/libpq/nls.mk | 12 +-
src/makefiles/meson.build | 2 +
src/test/modules/oauth_validator/meson.build | 2 +-
.../modules/oauth_validator/t/002_client.pl | 2 +-
25 files changed, 1080 insertions(+), 140 deletions(-)
create mode 100644 src/interfaces/libpq-oauth/Makefile
create mode 100644 src/interfaces/libpq-oauth/README
create mode 100644 src/interfaces/libpq-oauth/exports.txt
create mode 100644 src/interfaces/libpq-oauth/meson.build
rename src/interfaces/{libpq/fe-auth-oauth-curl.c => libpq-oauth/oauth-curl.c} (94%)
create mode 100644 src/interfaces/libpq-oauth/oauth-curl.h
create mode 100644 src/interfaces/libpq-oauth/oauth-utils.c
create mode 100644 src/interfaces/libpq-oauth/oauth-utils.h
diff --git a/config/programs.m4 b/config/programs.m4
index 0a07feb37cc..0ad1e58b48d 100644
--- a/config/programs.m4
+++ b/config/programs.m4
@@ -286,9 +286,20 @@ AC_DEFUN([PGAC_CHECK_LIBCURL],
[
AC_CHECK_HEADER(curl/curl.h, [],
[AC_MSG_ERROR([header file <curl/curl.h> is required for --with-libcurl])])
- AC_CHECK_LIB(curl, curl_multi_init, [],
+ AC_CHECK_LIB(curl, curl_multi_init, [
+ AC_DEFINE([HAVE_LIBCURL], [1], [Define to 1 if you have the `curl' library (-lcurl).])
+ AC_SUBST(LIBCURL_LDLIBS, -lcurl)
+ ],
[AC_MSG_ERROR([library 'curl' does not provide curl_multi_init])])
+ pgac_save_CPPFLAGS=$CPPFLAGS
+ pgac_save_LDFLAGS=$LDFLAGS
+ pgac_save_LIBS=$LIBS
+
+ CPPFLAGS="$LIBCURL_CPPFLAGS $CPPFLAGS"
+ LDFLAGS="$LIBCURL_LDFLAGS $LDFLAGS"
+ LIBS="$LIBCURL_LDLIBS $LIBS"
+
# Check to see whether the current platform supports threadsafe Curl
# initialization.
AC_CACHE_CHECK([for curl_global_init thread safety], [pgac_cv__libcurl_threadsafe_init],
@@ -338,4 +349,8 @@ AC_DEFUN([PGAC_CHECK_LIBCURL],
*** lookups. Rebuild libcurl with the AsynchDNS feature enabled in order
*** to use it with libpq.])
fi
+
+ CPPFLAGS=$pgac_save_CPPFLAGS
+ LDFLAGS=$pgac_save_LDFLAGS
+ LIBS=$pgac_save_LIBS
])# PGAC_CHECK_LIBCURL
diff --git a/configure b/configure
index 0936010718d..a4c4bcb40ea 100755
--- a/configure
+++ b/configure
@@ -655,6 +655,7 @@ UUID_LIBS
LDAP_LIBS_BE
LDAP_LIBS_FE
with_ssl
+LIBCURL_LDLIBS
PTHREAD_CFLAGS
PTHREAD_LIBS
PTHREAD_CC
@@ -711,6 +712,8 @@ with_libxml
LIBNUMA_LIBS
LIBNUMA_CFLAGS
with_libnuma
+LIBCURL_LDFLAGS
+LIBCURL_CPPFLAGS
LIBCURL_LIBS
LIBCURL_CFLAGS
with_libcurl
@@ -9053,19 +9056,27 @@ $as_echo "yes" >&6; }
fi
- # We only care about -I, -D, and -L switches;
- # note that -lcurl will be added by PGAC_CHECK_LIBCURL below.
+ # Curl's flags are kept separate from the standard CPPFLAGS/LDFLAGS. We use
+ # them only for libpq-oauth.
+ LIBCURL_CPPFLAGS=
+ LIBCURL_LDFLAGS=
+
+ # We only care about -I, -D, and -L switches. Note that -lcurl will be added
+ # to LIBCURL_LDLIBS by PGAC_CHECK_LIBCURL, below.
for pgac_option in $LIBCURL_CFLAGS; do
case $pgac_option in
- -I*|-D*) CPPFLAGS="$CPPFLAGS $pgac_option";;
+ -I*|-D*) LIBCURL_CPPFLAGS="$LIBCURL_CPPFLAGS $pgac_option";;
esac
done
for pgac_option in $LIBCURL_LIBS; do
case $pgac_option in
- -L*) LDFLAGS="$LDFLAGS $pgac_option";;
+ -L*) LIBCURL_LDFLAGS="$LIBCURL_LDFLAGS $pgac_option";;
esac
done
+
+
+
# OAuth requires python for testing
if test "$with_python" != yes; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** OAuth support tests require --with-python to run" >&5
@@ -12704,9 +12715,6 @@ fi
fi
-# XXX libcurl must link after libgssapi_krb5 on FreeBSD to avoid segfaults
-# during gss_acquire_cred(). This is possibly related to Curl's Heimdal
-# dependency on that platform?
if test "$with_libcurl" = yes ; then
ac_fn_c_check_header_mongrel "$LINENO" "curl/curl.h" "ac_cv_header_curl_curl_h" "$ac_includes_default"
@@ -12754,17 +12762,26 @@ fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_multi_init" >&5
$as_echo "$ac_cv_lib_curl_curl_multi_init" >&6; }
if test "x$ac_cv_lib_curl_curl_multi_init" = xyes; then :
- cat >>confdefs.h <<_ACEOF
-#define HAVE_LIBCURL 1
-_ACEOF
- LIBS="-lcurl $LIBS"
+
+$as_echo "#define HAVE_LIBCURL 1" >>confdefs.h
+
+ LIBCURL_LDLIBS=-lcurl
+
else
as_fn_error $? "library 'curl' does not provide curl_multi_init" "$LINENO" 5
fi
+ pgac_save_CPPFLAGS=$CPPFLAGS
+ pgac_save_LDFLAGS=$LDFLAGS
+ pgac_save_LIBS=$LIBS
+
+ CPPFLAGS="$LIBCURL_CPPFLAGS $CPPFLAGS"
+ LDFLAGS="$LIBCURL_LDFLAGS $LDFLAGS"
+ LIBS="$LIBCURL_LDLIBS $LIBS"
+
# Check to see whether the current platform supports threadsafe Curl
# initialization.
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for curl_global_init thread safety" >&5
@@ -12868,6 +12885,10 @@ $as_echo "$pgac_cv__libcurl_async_dns" >&6; }
*** to use it with libpq." "$LINENO" 5
fi
+ CPPFLAGS=$pgac_save_CPPFLAGS
+ LDFLAGS=$pgac_save_LDFLAGS
+ LIBS=$pgac_save_LIBS
+
fi
if test "$with_gssapi" = yes ; then
@@ -14516,6 +14537,13 @@ done
fi
+if test "$with_libcurl" = yes ; then
+ # Error out early if this platform can't support libpq-oauth.
+ if test "$ac_cv_header_sys_event_h" != yes -a "$ac_cv_header_sys_epoll_h" != yes; then
+ as_fn_error $? "client OAuth is not supported on this platform" "$LINENO" 5
+ fi
+fi
+
##
## Types, structures, compiler characteristics
##
diff --git a/configure.ac b/configure.ac
index 2a78cddd825..c0471030e90 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1033,19 +1033,27 @@ if test "$with_libcurl" = yes ; then
# to explicitly set TLS 1.3 ciphersuites).
PKG_CHECK_MODULES(LIBCURL, [libcurl >= 7.61.0])
- # We only care about -I, -D, and -L switches;
- # note that -lcurl will be added by PGAC_CHECK_LIBCURL below.
+ # Curl's flags are kept separate from the standard CPPFLAGS/LDFLAGS. We use
+ # them only for libpq-oauth.
+ LIBCURL_CPPFLAGS=
+ LIBCURL_LDFLAGS=
+
+ # We only care about -I, -D, and -L switches. Note that -lcurl will be added
+ # to LIBCURL_LDLIBS by PGAC_CHECK_LIBCURL, below.
for pgac_option in $LIBCURL_CFLAGS; do
case $pgac_option in
- -I*|-D*) CPPFLAGS="$CPPFLAGS $pgac_option";;
+ -I*|-D*) LIBCURL_CPPFLAGS="$LIBCURL_CPPFLAGS $pgac_option";;
esac
done
for pgac_option in $LIBCURL_LIBS; do
case $pgac_option in
- -L*) LDFLAGS="$LDFLAGS $pgac_option";;
+ -L*) LIBCURL_LDFLAGS="$LIBCURL_LDFLAGS $pgac_option";;
esac
done
+ AC_SUBST(LIBCURL_CPPFLAGS)
+ AC_SUBST(LIBCURL_LDFLAGS)
+
# OAuth requires python for testing
if test "$with_python" != yes; then
AC_MSG_WARN([*** OAuth support tests require --with-python to run])
@@ -1354,9 +1362,6 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
-# XXX libcurl must link after libgssapi_krb5 on FreeBSD to avoid segfaults
-# during gss_acquire_cred(). This is possibly related to Curl's Heimdal
-# dependency on that platform?
if test "$with_libcurl" = yes ; then
PGAC_CHECK_LIBCURL
fi
@@ -1654,6 +1659,13 @@ if test "$PORTNAME" = "win32" ; then
AC_CHECK_HEADERS(crtdefs.h)
fi
+if test "$with_libcurl" = yes ; then
+ # Error out early if this platform can't support libpq-oauth.
+ if test "$ac_cv_header_sys_event_h" != yes -a "$ac_cv_header_sys_epoll_h" != yes; then
+ AC_MSG_ERROR([client-side OAuth is not supported on this platform])
+ fi
+fi
+
##
## Types, structures, compiler characteristics
##
diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml
index 077bcc20759..d928b103d22 100644
--- a/doc/src/sgml/installation.sgml
+++ b/doc/src/sgml/installation.sgml
@@ -313,6 +313,14 @@
</para>
</listitem>
+ <listitem>
+ <para>
+ You need <productname>Curl</productname> to build an optional module
+ which implements the <link linkend="libpq-oauth">OAuth Device
+ Authorization flow</link> for client applications.
+ </para>
+ </listitem>
+
<listitem>
<para>
You need <productname>LZ4</productname>, if you want to support
diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml
index 8cdd2997d43..695fe958c3e 100644
--- a/doc/src/sgml/libpq.sgml
+++ b/doc/src/sgml/libpq.sgml
@@ -10226,15 +10226,20 @@ void PQinitSSL(int do_ssl);
<title>OAuth Support</title>
<para>
- libpq implements support for the OAuth v2 Device Authorization client flow,
+ <application>libpq</application> implements support for the OAuth v2 Device Authorization client flow,
documented in
<ulink url="https://datatracker.ietf.org/doc/html/rfc8628">RFC 8628</ulink>,
- which it will attempt to use by default if the server
+ as an optional module. See the <link linkend="configure-option-with-libcurl">
+ installation documentation</link> for information on how to enable support
+ for Device Authorization as a builtin flow.
+ </para>
+ <para>
+ When support is enabled and the optional module installed, <application>libpq</application>
+ will use the builtin flow by default if the server
<link linkend="auth-oauth">requests a bearer token</link> during
authentication. This flow can be utilized even if the system running the
client application does not have a usable web browser, for example when
- running a client via <application>SSH</application>. Client applications may implement their own flows
- instead; see <xref linkend="libpq-oauth-authdata-hooks"/>.
+ running a client via <acronym>SSH</acronym>.
</para>
<para>
The builtin flow will, by default, print a URL to visit and a user code to
@@ -10251,6 +10256,11 @@ Visit https://example.com/device and enter the code: ABCD-EFGH
they match expectations, before continuing. Permissions should not be given
to untrusted third parties.
</para>
+ <para>
+ Client applications may implement their own flows to customize interaction
+ and integration with applications. See <xref linkend="libpq-oauth-authdata-hooks"/>
+ for more information on how add a custom flow to <application>libpq</application>.
+ </para>
<para>
For an OAuth client flow to be usable, the connection string must at minimum
contain <xref linkend="libpq-connect-oauth-issuer"/> and
@@ -10366,7 +10376,9 @@ typedef struct _PGpromptOAuthDevice
</synopsis>
</para>
<para>
- The OAuth Device Authorization flow included in <application>libpq</application>
+ The OAuth Device Authorization flow which
+ <link linkend="configure-option-with-libcurl">can be included</link>
+ in <application>libpq</application>
requires the end user to visit a URL with a browser, then enter a code
which permits <application>libpq</application> to connect to the server
on their behalf. The default prompt simply prints the
@@ -10378,7 +10390,8 @@ typedef struct _PGpromptOAuthDevice
This callback is only invoked during the builtin device
authorization flow. If the application installs a
<link linkend="libpq-oauth-authdata-oauth-bearer-token">custom OAuth
- flow</link>, this authdata type will not be used.
+ flow</link>, or <application>libpq</application> was not built with
+ support for the builtin flow, this authdata type will not be used.
</para>
<para>
If a non-NULL <structfield>verification_uri_complete</structfield> is
@@ -10400,8 +10413,9 @@ typedef struct _PGpromptOAuthDevice
</term>
<listitem>
<para>
- Replaces the entire OAuth flow with a custom implementation. The hook
- should either directly return a Bearer token for the current
+ Adds a custom implementation of a flow, replacing the builtin flow if
+ it is <link linkend="configure-option-with-libcurl">installed</link>.
+ The hook should either directly return a Bearer token for the current
user/issuer/scope combination, if one is available without blocking, or
else set up an asynchronous callback to retrieve one.
</para>
diff --git a/meson.build b/meson.build
index a1516e54529..29d46c8ad01 100644
--- a/meson.build
+++ b/meson.build
@@ -107,6 +107,7 @@ os_deps = []
backend_both_deps = []
backend_deps = []
libpq_deps = []
+libpq_oauth_deps = []
pg_sysroot = ''
@@ -860,13 +861,13 @@ endif
###############################################################
libcurlopt = get_option('libcurl')
+oauth_flow_supported = false
+
if not libcurlopt.disabled()
# Check for libcurl 7.61.0 or higher (corresponding to RHEL8 and the ability
# to explicitly set TLS 1.3 ciphersuites).
libcurl = dependency('libcurl', version: '>= 7.61.0', required: libcurlopt)
if libcurl.found()
- cdata.set('USE_LIBCURL', 1)
-
# Check to see whether the current platform supports thread-safe Curl
# initialization.
libcurl_threadsafe_init = false
@@ -938,6 +939,22 @@ if not libcurlopt.disabled()
endif
endif
+ # Check that the current platform supports our builtin flow. This requires
+ # libcurl and one of either epoll or kqueue.
+ oauth_flow_supported = (
+ libcurl.found()
+ and (cc.check_header('sys/event.h', required: false,
+ args: test_c_args, include_directories: postgres_inc)
+ or cc.check_header('sys/epoll.h', required: false,
+ args: test_c_args, include_directories: postgres_inc))
+ )
+
+ if oauth_flow_supported
+ cdata.set('USE_LIBCURL', 1)
+ elif libcurlopt.enabled()
+ error('client-side OAuth is not supported on this platform')
+ endif
+
else
libcurl = not_found_dep
endif
@@ -3272,17 +3289,18 @@ libpq_deps += [
gssapi,
ldap_r,
- # XXX libcurl must link after libgssapi_krb5 on FreeBSD to avoid segfaults
- # during gss_acquire_cred(). This is possibly related to Curl's Heimdal
- # dependency on that platform?
- libcurl,
libintl,
ssl,
]
+libpq_oauth_deps += [
+ libcurl,
+]
+
subdir('src/interfaces/libpq')
-# fe_utils depends on libpq
+# fe_utils and libpq-oauth depends on libpq
subdir('src/fe_utils')
+subdir('src/interfaces/libpq-oauth')
# for frontend binaries
frontend_code = declare_dependency(
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index 6722fbdf365..04952b533de 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -347,6 +347,9 @@ perl_embed_ldflags = @perl_embed_ldflags@
AWK = @AWK@
LN_S = @LN_S@
+LIBCURL_CPPFLAGS = @LIBCURL_CPPFLAGS@
+LIBCURL_LDFLAGS = @LIBCURL_LDFLAGS@
+LIBCURL_LDLIBS = @LIBCURL_LDLIBS@
MSGFMT = @MSGFMT@
MSGFMT_FLAGS = @MSGFMT_FLAGS@
MSGMERGE = @MSGMERGE@
diff --git a/src/interfaces/Makefile b/src/interfaces/Makefile
index 7d56b29d28f..e6822caa206 100644
--- a/src/interfaces/Makefile
+++ b/src/interfaces/Makefile
@@ -14,7 +14,19 @@ include $(top_builddir)/src/Makefile.global
SUBDIRS = libpq ecpg
+ifeq ($(with_libcurl), yes)
+SUBDIRS += libpq-oauth
+else
+ALWAYS_SUBDIRS += libpq-oauth
+endif
+
$(recurse)
+$(recurse_always)
all-ecpg-recurse: all-libpq-recurse
install-ecpg-recurse: install-libpq-recurse
+
+ifeq ($(with_libcurl), yes)
+all-libpq-oauth-recurse: all-libpq-recurse
+install-libpq-oauth-recurse: install-libpq-recurse
+endif
diff --git a/src/interfaces/libpq-oauth/Makefile b/src/interfaces/libpq-oauth/Makefile
new file mode 100644
index 00000000000..3e4b34142e0
--- /dev/null
+++ b/src/interfaces/libpq-oauth/Makefile
@@ -0,0 +1,83 @@
+#-------------------------------------------------------------------------
+#
+# Makefile for libpq-oauth
+#
+# Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+# Portions Copyright (c) 1994, Regents of the University of California
+#
+# src/interfaces/libpq-oauth/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/interfaces/libpq-oauth
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+PGFILEDESC = "libpq-oauth - device authorization OAuth support"
+
+# This is an internal module; we don't want an SONAME and therefore do not set
+# SO_MAJOR_VERSION.
+NAME = pq-oauth-$(MAJORVERSION)
+
+# Force the name "libpq-oauth" for both the static and shared libraries. The
+# staticlib doesn't need version information in its name.
+override shlib := lib$(NAME)$(DLSUFFIX)
+override stlib := libpq-oauth.a
+
+override CPPFLAGS := -I$(libpq_srcdir) -I$(top_builddir)/src/port $(LIBCURL_CPPFLAGS) $(CPPFLAGS)
+
+OBJS = \
+ $(WIN32RES)
+
+OBJS_STATIC = oauth-curl.o
+
+# The shared library needs additional glue symbols.
+OBJS_SHLIB = \
+ oauth-curl_shlib.o \
+ oauth-utils.o \
+
+oauth-utils.o: override CPPFLAGS += -DUSE_DYNAMIC_OAUTH
+oauth-curl_shlib.o: override CPPFLAGS_SHLIB += -DUSE_DYNAMIC_OAUTH
+
+# Add shlib-/stlib-specific objects.
+$(shlib): override OBJS += $(OBJS_SHLIB)
+$(shlib): $(OBJS_SHLIB)
+
+$(stlib): override OBJS += $(OBJS_STATIC)
+$(stlib): $(OBJS_STATIC)
+
+SHLIB_LINK_INTERNAL = $(libpq_pgport_shlib)
+SHLIB_LINK = $(LIBCURL_LDFLAGS) $(LIBCURL_LDLIBS)
+SHLIB_PREREQS = submake-libpq
+SHLIB_EXPORTS = exports.txt
+
+# Disable -bundle_loader on macOS.
+BE_DLLLIBS =
+
+# By default, a library without an SONAME doesn't get a static library, so we
+# add it to the build explicitly.
+all: all-lib all-static-lib
+
+# Shared library stuff
+include $(top_srcdir)/src/Makefile.shlib
+
+# Use src/common/Makefile's trick for tracking dependencies of shlib-specific
+# objects.
+%_shlib.o: %.c %.o
+ $(CC) $(CFLAGS) $(CFLAGS_SL) $(CPPFLAGS) $(CPPFLAGS_SHLIB) -c $< -o $@
+
+# Ignore the standard rules for SONAME-less installation; we want both the
+# static and shared libraries to go into libdir.
+install: all installdirs $(stlib) $(shlib)
+ $(INSTALL_SHLIB) $(shlib) '$(DESTDIR)$(libdir)/$(shlib)'
+ $(INSTALL_STLIB) $(stlib) '$(DESTDIR)$(libdir)/$(stlib)'
+
+installdirs:
+ $(MKDIR_P) '$(DESTDIR)$(libdir)'
+
+uninstall:
+ rm -f '$(DESTDIR)$(libdir)/$(stlib)'
+ rm -f '$(DESTDIR)$(libdir)/$(shlib)'
+
+clean distclean: clean-lib
+ rm -f $(OBJS) $(OBJS_STATIC) $(OBJS_SHLIB)
diff --git a/src/interfaces/libpq-oauth/README b/src/interfaces/libpq-oauth/README
new file mode 100644
index 00000000000..4579b45c0f9
--- /dev/null
+++ b/src/interfaces/libpq-oauth/README
@@ -0,0 +1,58 @@
+libpq-oauth is an optional module implementing the Device Authorization flow for
+OAuth clients (RFC 8628). It was originally developed as part of libpq core and
+later split out as its own shared library in order to isolate its dependency on
+libcurl. (End users who don't want the Curl dependency can simply choose not to
+install this module.)
+
+If a connection string allows the use of OAuth, and the server asks for it, and
+a libpq client has not installed its own custom OAuth flow, libpq will attempt
+to delay-load this module using dlopen() and the following ABI. Failure to load
+results in a failed connection.
+
+= Load-Time ABI =
+
+This module ABI is an internal implementation detail, so it's subject to change
+across major releases; the name of the module (libpq-oauth-MAJOR) reflects this.
+The module exports the following symbols:
+
+- PostgresPollingStatusType pg_fe_run_oauth_flow(PGconn *conn);
+- void pg_fe_cleanup_oauth_flow(PGconn *conn);
+
+pg_fe_run_oauth_flow and pg_fe_cleanup_oauth_flow are implementations of
+conn->async_auth and conn->cleanup_async_auth, respectively.
+
+- void libpq_oauth_init(pgthreadlock_t threadlock,
+ libpq_gettext_func gettext_impl,
+ conn_errorMessage_func errmsg_impl,
+ conn_oauth_client_id_func clientid_impl,
+ conn_oauth_client_secret_func clientsecret_impl,
+ conn_oauth_discovery_uri_func discoveryuri_impl,
+ conn_oauth_issuer_id_func issuerid_impl,
+ conn_oauth_scope_func scope_impl,
+ conn_sasl_state_func saslstate_impl,
+ set_conn_altsock_func setaltsock_impl,
+ set_conn_oauth_token_func settoken_impl);
+
+At the moment, pg_fe_run_oauth_flow() relies on libpq's pg_g_threadlock and
+libpq_gettext(), which must be injected by libpq using this initialization
+function before the flow is run.
+
+It also relies on access to several members of the PGconn struct. Not only can
+these change positions across minor versions, but the offsets aren't necessarily
+stable within a single minor release (conn->errorMessage, for instance, can
+change offsets depending on configure-time options). Therefore the necessary
+accessors (named conn_*) and mutators (set_conn_*) are injected here. With this
+approach, we can safely search the standard dlopen() paths (e.g. RPATH,
+LD_LIBRARY_PATH, the SO cache) for an implementation module to use, even if that
+module wasn't compiled at the same time as libpq -- which becomes especially
+important during "live upgrade" situations where a running libpq application has
+the libpq-oauth module updated out from under it before it's first loaded from
+disk.
+
+= Static Build =
+
+The static library libpq.a does not perform any dynamic loading. If the builtin
+flow is enabled, the application is expected to link against libpq-oauth.a
+directly to provide the necessary symbols. (libpq.a and libpq-oauth.a must be
+part of the same build. Unlike the dynamic module, there are no translation
+shims provided.)
diff --git a/src/interfaces/libpq-oauth/exports.txt b/src/interfaces/libpq-oauth/exports.txt
new file mode 100644
index 00000000000..6891a83dbf9
--- /dev/null
+++ b/src/interfaces/libpq-oauth/exports.txt
@@ -0,0 +1,4 @@
+# src/interfaces/libpq-oauth/exports.txt
+libpq_oauth_init 1
+pg_fe_run_oauth_flow 2
+pg_fe_cleanup_oauth_flow 3
diff --git a/src/interfaces/libpq-oauth/meson.build b/src/interfaces/libpq-oauth/meson.build
new file mode 100644
index 00000000000..9e7301a7f63
--- /dev/null
+++ b/src/interfaces/libpq-oauth/meson.build
@@ -0,0 +1,45 @@
+# Copyright (c) 2022-2025, PostgreSQL Global Development Group
+
+if not oauth_flow_supported
+ subdir_done()
+endif
+
+libpq_oauth_sources = files(
+ 'oauth-curl.c',
+)
+
+# The shared library needs additional glue symbols.
+libpq_oauth_so_sources = files(
+ 'oauth-utils.c',
+)
+libpq_oauth_so_c_args = ['-DUSE_DYNAMIC_OAUTH']
+
+export_file = custom_target('libpq-oauth.exports',
+ kwargs: gen_export_kwargs,
+)
+
+# port needs to be in include path due to pthread-win32.h
+libpq_oauth_inc = include_directories('.', '../libpq', '../../port')
+
+libpq_oauth_st = static_library('libpq-oauth',
+ libpq_oauth_sources,
+ include_directories: [libpq_oauth_inc, postgres_inc],
+ c_pch: pch_postgres_fe_h,
+ dependencies: [frontend_stlib_code, libpq_oauth_deps],
+ kwargs: default_lib_args,
+)
+
+# This is an internal module; we don't want an SONAME and therefore do not set
+# SO_MAJOR_VERSION.
+libpq_oauth_name = 'libpq-oauth-@0@'.format(pg_version_major)
+
+libpq_oauth_so = shared_module(libpq_oauth_name,
+ libpq_oauth_sources + libpq_oauth_so_sources,
+ include_directories: [libpq_oauth_inc, postgres_inc],
+ c_args: libpq_so_c_args,
+ c_pch: pch_postgres_fe_h,
+ dependencies: [frontend_shlib_code, libpq, libpq_oauth_deps],
+ link_depends: export_file,
+ link_args: export_fmt.format(export_file.full_path()),
+ kwargs: default_lib_args,
+)
diff --git a/src/interfaces/libpq/fe-auth-oauth-curl.c b/src/interfaces/libpq-oauth/oauth-curl.c
similarity index 94%
rename from src/interfaces/libpq/fe-auth-oauth-curl.c
rename to src/interfaces/libpq-oauth/oauth-curl.c
index c195e00cd28..3acdcc52de8 100644
--- a/src/interfaces/libpq/fe-auth-oauth-curl.c
+++ b/src/interfaces/libpq-oauth/oauth-curl.c
@@ -1,6 +1,6 @@
/*-------------------------------------------------------------------------
*
- * fe-auth-oauth-curl.c
+ * oauth-curl.c
* The libcurl implementation of OAuth/OIDC authentication, using the
* OAuth Device Authorization Grant (RFC 8628).
*
@@ -8,7 +8,7 @@
* Portions Copyright (c) 1994, Regents of the University of California
*
* IDENTIFICATION
- * src/interfaces/libpq/fe-auth-oauth-curl.c
+ * src/interfaces/libpq-oauth/oauth-curl.c
*
*-------------------------------------------------------------------------
*/
@@ -17,20 +17,56 @@
#include <curl/curl.h>
#include <math.h>
-#ifdef HAVE_SYS_EPOLL_H
+#include <unistd.h>
+
+#if defined(HAVE_SYS_EPOLL_H)
#include <sys/epoll.h>
#include <sys/timerfd.h>
-#endif
-#ifdef HAVE_SYS_EVENT_H
+#elif defined(HAVE_SYS_EVENT_H)
#include <sys/event.h>
+#else
+#error libpq-oauth is not supported on this platform
#endif
-#include <unistd.h>
#include "common/jsonapi.h"
-#include "fe-auth.h"
#include "fe-auth-oauth.h"
-#include "libpq-int.h"
#include "mb/pg_wchar.h"
+#include "oauth-curl.h"
+
+#ifdef USE_DYNAMIC_OAUTH
+
+/*
+ * The module build is decoupled from libpq-int.h, to try to avoid inadvertent
+ * ABI breaks during minor version bumps. Replacements for the missing internals
+ * are provided by oauth-utils.
+ */
+#include "oauth-utils.h"
+
+#else /* !USE_DYNAMIC_OAUTH */
+
+/*
+ * Static builds may rely on PGconn offsets directly. Keep these aligned with
+ * the bank of callbacks in oauth-utils.h.
+ */
+#include "libpq-int.h"
+
+#define conn_errorMessage(CONN) (&CONN->errorMessage)
+#define conn_oauth_client_id(CONN) (CONN->oauth_client_id)
+#define conn_oauth_client_secret(CONN) (CONN->oauth_client_secret)
+#define conn_oauth_discovery_uri(CONN) (CONN->oauth_discovery_uri)
+#define conn_oauth_issuer_id(CONN) (CONN->oauth_issuer_id)
+#define conn_oauth_scope(CONN) (CONN->oauth_scope)
+#define conn_sasl_state(CONN) (CONN->sasl_state)
+
+#define set_conn_altsock(CONN, VAL) do { CONN->altsock = VAL; } while (0)
+#define set_conn_oauth_token(CONN, VAL) do { CONN->oauth_token = VAL; } while (0)
+
+#endif /* !USE_DYNAMIC_OAUTH */
+
+/* One final guardrail against accidental inclusion... */
+#if defined(USE_DYNAMIC_OAUTH) && defined(LIBPQ_INT_H)
+#error do not rely on libpq-int.h in libpq-oauth.so
+#endif
/*
* It's generally prudent to set a maximum response size to buffer in memory,
@@ -303,7 +339,7 @@ free_async_ctx(PGconn *conn, struct async_ctx *actx)
void
pg_fe_cleanup_oauth_flow(PGconn *conn)
{
- fe_oauth_state *state = conn->sasl_state;
+ fe_oauth_state *state = conn_sasl_state(conn);
if (state->async_ctx)
{
@@ -311,7 +347,7 @@ pg_fe_cleanup_oauth_flow(PGconn *conn)
state->async_ctx = NULL;
}
- conn->altsock = PGINVALID_SOCKET;
+ set_conn_altsock(conn, PGINVALID_SOCKET);
}
/*
@@ -1110,7 +1146,7 @@ parse_access_token(struct async_ctx *actx, struct token *tok)
static bool
setup_multiplexer(struct async_ctx *actx)
{
-#ifdef HAVE_SYS_EPOLL_H
+#if defined(HAVE_SYS_EPOLL_H)
struct epoll_event ev = {.events = EPOLLIN};
actx->mux = epoll_create1(EPOLL_CLOEXEC);
@@ -1134,8 +1170,7 @@ setup_multiplexer(struct async_ctx *actx)
}
return true;
-#endif
-#ifdef HAVE_SYS_EVENT_H
+#elif defined(HAVE_SYS_EVENT_H)
actx->mux = kqueue();
if (actx->mux < 0)
{
@@ -1158,10 +1193,9 @@ setup_multiplexer(struct async_ctx *actx)
}
return true;
+#else
+#error setup_multiplexer is not implemented on this platform
#endif
-
- actx_error(actx, "libpq does not support the Device Authorization flow on this platform");
- return false;
}
/*
@@ -1174,7 +1208,7 @@ register_socket(CURL *curl, curl_socket_t socket, int what, void *ctx,
{
struct async_ctx *actx = ctx;
-#ifdef HAVE_SYS_EPOLL_H
+#if defined(HAVE_SYS_EPOLL_H)
struct epoll_event ev = {0};
int res;
int op = EPOLL_CTL_ADD;
@@ -1230,8 +1264,7 @@ register_socket(CURL *curl, curl_socket_t socket, int what, void *ctx,
}
return 0;
-#endif
-#ifdef HAVE_SYS_EVENT_H
+#elif defined(HAVE_SYS_EVENT_H)
struct kevent ev[2] = {0};
struct kevent ev_out[2];
struct timespec timeout = {0};
@@ -1312,10 +1345,9 @@ register_socket(CURL *curl, curl_socket_t socket, int what, void *ctx,
}
return 0;
+#else
+#error register_socket is not implemented on this platform
#endif
-
- actx_error(actx, "libpq does not support multiplexer sockets on this platform");
- return -1;
}
/*
@@ -1334,7 +1366,7 @@ register_socket(CURL *curl, curl_socket_t socket, int what, void *ctx,
static bool
set_timer(struct async_ctx *actx, long timeout)
{
-#if HAVE_SYS_EPOLL_H
+#if defined(HAVE_SYS_EPOLL_H)
struct itimerspec spec = {0};
if (timeout < 0)
@@ -1363,8 +1395,7 @@ set_timer(struct async_ctx *actx, long timeout)
}
return true;
-#endif
-#ifdef HAVE_SYS_EVENT_H
+#elif defined(HAVE_SYS_EVENT_H)
struct kevent ev;
#ifdef __NetBSD__
@@ -1419,10 +1450,9 @@ set_timer(struct async_ctx *actx, long timeout)
}
return true;
+#else
+#error set_timer is not implemented on this platform
#endif
-
- actx_error(actx, "libpq does not support timers on this platform");
- return false;
}
/*
@@ -1433,7 +1463,7 @@ set_timer(struct async_ctx *actx, long timeout)
static int
timer_expired(struct async_ctx *actx)
{
-#if HAVE_SYS_EPOLL_H
+#if defined(HAVE_SYS_EPOLL_H)
struct itimerspec spec = {0};
if (timerfd_gettime(actx->timerfd, &spec) < 0)
@@ -1453,8 +1483,7 @@ timer_expired(struct async_ctx *actx)
/* If the remaining time to expiration is zero, we're done. */
return (spec.it_value.tv_sec == 0
&& spec.it_value.tv_nsec == 0);
-#endif
-#ifdef HAVE_SYS_EVENT_H
+#elif defined(HAVE_SYS_EVENT_H)
int res;
/* Is the timer queue ready? */
@@ -1466,10 +1495,9 @@ timer_expired(struct async_ctx *actx)
}
return (res > 0);
+#else
+#error timer_expired is not implemented on this platform
#endif
-
- actx_error(actx, "libpq does not support timers on this platform");
- return -1;
}
/*
@@ -2070,8 +2098,9 @@ static bool
check_issuer(struct async_ctx *actx, PGconn *conn)
{
const struct provider *provider = &actx->provider;
+ const char *oauth_issuer_id = conn_oauth_issuer_id(conn);
- Assert(conn->oauth_issuer_id); /* ensured by setup_oauth_parameters() */
+ Assert(oauth_issuer_id); /* ensured by setup_oauth_parameters() */
Assert(provider->issuer); /* ensured by parse_provider() */
/*---
@@ -2091,11 +2120,11 @@ check_issuer(struct async_ctx *actx, PGconn *conn)
* sent to. This comparison MUST use simple string comparison as defined
* in Section 6.2.1 of [RFC3986].
*/
- if (strcmp(conn->oauth_issuer_id, provider->issuer) != 0)
+ if (strcmp(oauth_issuer_id, provider->issuer) != 0)
{
actx_error(actx,
"the issuer identifier (%s) does not match oauth_issuer (%s)",
- provider->issuer, conn->oauth_issuer_id);
+ provider->issuer, oauth_issuer_id);
return false;
}
@@ -2172,11 +2201,14 @@ check_for_device_flow(struct async_ctx *actx)
static bool
add_client_identification(struct async_ctx *actx, PQExpBuffer reqbody, PGconn *conn)
{
+ const char *oauth_client_id = conn_oauth_client_id(conn);
+ const char *oauth_client_secret = conn_oauth_client_secret(conn);
+
bool success = false;
char *username = NULL;
char *password = NULL;
- if (conn->oauth_client_secret) /* Zero-length secrets are permitted! */
+ if (oauth_client_secret) /* Zero-length secrets are permitted! */
{
/*----
* Use HTTP Basic auth to send the client_id and secret. Per RFC 6749,
@@ -2204,8 +2236,8 @@ add_client_identification(struct async_ctx *actx, PQExpBuffer reqbody, PGconn *c
* would it be redundant, but some providers in the wild (e.g. Okta)
* refuse to accept it.
*/
- username = urlencode(conn->oauth_client_id);
- password = urlencode(conn->oauth_client_secret);
+ username = urlencode(oauth_client_id);
+ password = urlencode(oauth_client_secret);
if (!username || !password)
{
@@ -2225,7 +2257,7 @@ add_client_identification(struct async_ctx *actx, PQExpBuffer reqbody, PGconn *c
* If we're not otherwise authenticating, client_id is REQUIRED in the
* request body.
*/
- build_urlencoded(reqbody, "client_id", conn->oauth_client_id);
+ build_urlencoded(reqbody, "client_id", oauth_client_id);
CHECK_SETOPT(actx, CURLOPT_HTTPAUTH, CURLAUTH_NONE, goto cleanup);
actx->used_basic_auth = false;
@@ -2253,16 +2285,17 @@ cleanup:
static bool
start_device_authz(struct async_ctx *actx, PGconn *conn)
{
+ const char *oauth_scope = conn_oauth_scope(conn);
const char *device_authz_uri = actx->provider.device_authorization_endpoint;
PQExpBuffer work_buffer = &actx->work_data;
- Assert(conn->oauth_client_id); /* ensured by setup_oauth_parameters() */
+ Assert(conn_oauth_client_id(conn)); /* ensured by setup_oauth_parameters() */
Assert(device_authz_uri); /* ensured by check_for_device_flow() */
/* Construct our request body. */
resetPQExpBuffer(work_buffer);
- if (conn->oauth_scope && conn->oauth_scope[0])
- build_urlencoded(work_buffer, "scope", conn->oauth_scope);
+ if (oauth_scope && oauth_scope[0])
+ build_urlencoded(work_buffer, "scope", oauth_scope);
if (!add_client_identification(actx, work_buffer, conn))
return false;
@@ -2344,7 +2377,7 @@ start_token_request(struct async_ctx *actx, PGconn *conn)
const char *device_code = actx->authz.device_code;
PQExpBuffer work_buffer = &actx->work_data;
- Assert(conn->oauth_client_id); /* ensured by setup_oauth_parameters() */
+ Assert(conn_oauth_client_id(conn)); /* ensured by setup_oauth_parameters() */
Assert(token_uri); /* ensured by parse_provider() */
Assert(device_code); /* ensured by parse_device_authz() */
@@ -2487,8 +2520,9 @@ prompt_user(struct async_ctx *actx, PGconn *conn)
.verification_uri_complete = actx->authz.verification_uri_complete,
.expires_in = actx->authz.expires_in,
};
+ PQauthDataHook_type hook = PQgetAuthDataHook();
- res = PQauthDataHook(PQAUTHDATA_PROMPT_OAUTH_DEVICE, conn, &prompt);
+ res = hook(PQAUTHDATA_PROMPT_OAUTH_DEVICE, conn, &prompt);
if (!res)
{
@@ -2633,8 +2667,10 @@ done:
static PostgresPollingStatusType
pg_fe_run_oauth_flow_impl(PGconn *conn)
{
- fe_oauth_state *state = conn->sasl_state;
+ fe_oauth_state *state = conn_sasl_state(conn);
struct async_ctx *actx;
+ char *oauth_token = NULL;
+ PQExpBuffer errbuf;
if (!initialize_curl(conn))
return PGRES_POLLING_FAILED;
@@ -2676,7 +2712,7 @@ pg_fe_run_oauth_flow_impl(PGconn *conn)
do
{
/* By default, the multiplexer is the altsock. Reassign as desired. */
- conn->altsock = actx->mux;
+ set_conn_altsock(conn, actx->mux);
switch (actx->step)
{
@@ -2712,7 +2748,7 @@ pg_fe_run_oauth_flow_impl(PGconn *conn)
*/
if (!timer_expired(actx))
{
- conn->altsock = actx->timerfd;
+ set_conn_altsock(conn, actx->timerfd);
return PGRES_POLLING_READING;
}
@@ -2732,7 +2768,7 @@ pg_fe_run_oauth_flow_impl(PGconn *conn)
{
case OAUTH_STEP_INIT:
actx->errctx = "failed to fetch OpenID discovery document";
- if (!start_discovery(actx, conn->oauth_discovery_uri))
+ if (!start_discovery(actx, conn_oauth_discovery_uri(conn)))
goto error_return;
actx->step = OAUTH_STEP_DISCOVERY;
@@ -2768,9 +2804,15 @@ pg_fe_run_oauth_flow_impl(PGconn *conn)
break;
case OAUTH_STEP_TOKEN_REQUEST:
- if (!handle_token_response(actx, &conn->oauth_token))
+ if (!handle_token_response(actx, &oauth_token))
goto error_return;
+ /*
+ * Hook any oauth_token into the PGconn immediately so that
+ * the allocation isn't lost in case of an error.
+ */
+ set_conn_oauth_token(conn, oauth_token);
+
if (!actx->user_prompted)
{
/*
@@ -2783,7 +2825,7 @@ pg_fe_run_oauth_flow_impl(PGconn *conn)
actx->user_prompted = true;
}
- if (conn->oauth_token)
+ if (oauth_token)
break; /* done! */
/*
@@ -2798,7 +2840,7 @@ pg_fe_run_oauth_flow_impl(PGconn *conn)
* the client wait directly on the timerfd rather than the
* multiplexer.
*/
- conn->altsock = actx->timerfd;
+ set_conn_altsock(conn, actx->timerfd);
actx->step = OAUTH_STEP_WAIT_INTERVAL;
actx->running = 1;
@@ -2818,48 +2860,40 @@ pg_fe_run_oauth_flow_impl(PGconn *conn)
* point, actx->running will be set. But there are some corner cases
* where we can immediately loop back around; see start_request().
*/
- } while (!conn->oauth_token && !actx->running);
+ } while (!oauth_token && !actx->running);
/* If we've stored a token, we're done. Otherwise come back later. */
- return conn->oauth_token ? PGRES_POLLING_OK : PGRES_POLLING_READING;
+ return oauth_token ? PGRES_POLLING_OK : PGRES_POLLING_READING;
error_return:
+ errbuf = conn_errorMessage(conn);
/*
* Assemble the three parts of our error: context, body, and detail. See
* also the documentation for struct async_ctx.
*/
if (actx->errctx)
- {
- appendPQExpBufferStr(&conn->errorMessage,
- libpq_gettext(actx->errctx));
- appendPQExpBufferStr(&conn->errorMessage, ": ");
- }
+ appendPQExpBuffer(errbuf, "%s: ", libpq_gettext(actx->errctx));
if (PQExpBufferDataBroken(actx->errbuf))
- appendPQExpBufferStr(&conn->errorMessage,
- libpq_gettext("out of memory"));
+ appendPQExpBufferStr(errbuf, libpq_gettext("out of memory"));
else
- appendPQExpBufferStr(&conn->errorMessage, actx->errbuf.data);
+ appendPQExpBufferStr(errbuf, actx->errbuf.data);
if (actx->curl_err[0])
{
- size_t len;
-
- appendPQExpBuffer(&conn->errorMessage,
- " (libcurl: %s)", actx->curl_err);
+ appendPQExpBuffer(errbuf, " (libcurl: %s)", actx->curl_err);
/* Sometimes libcurl adds a newline to the error buffer. :( */
- len = conn->errorMessage.len;
- if (len >= 2 && conn->errorMessage.data[len - 2] == '\n')
+ if (errbuf->len >= 2 && errbuf->data[errbuf->len - 2] == '\n')
{
- conn->errorMessage.data[len - 2] = ')';
- conn->errorMessage.data[len - 1] = '\0';
- conn->errorMessage.len--;
+ errbuf->data[errbuf->len - 2] = ')';
+ errbuf->data[errbuf->len - 1] = '\0';
+ errbuf->len--;
}
}
- appendPQExpBufferChar(&conn->errorMessage, '\n');
+ appendPQExpBufferChar(errbuf, '\n');
return PGRES_POLLING_FAILED;
}
diff --git a/src/interfaces/libpq-oauth/oauth-curl.h b/src/interfaces/libpq-oauth/oauth-curl.h
new file mode 100644
index 00000000000..248d0424ad0
--- /dev/null
+++ b/src/interfaces/libpq-oauth/oauth-curl.h
@@ -0,0 +1,24 @@
+/*-------------------------------------------------------------------------
+ *
+ * oauth-curl.h
+ *
+ * Definitions for OAuth Device Authorization module
+ *
+ * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/interfaces/libpq-oauth/oauth-curl.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef OAUTH_CURL_H
+#define OAUTH_CURL_H
+
+#include "libpq-fe.h"
+
+/* Exported async-auth callbacks. */
+extern PGDLLEXPORT PostgresPollingStatusType pg_fe_run_oauth_flow(PGconn *conn);
+extern PGDLLEXPORT void pg_fe_cleanup_oauth_flow(PGconn *conn);
+
+#endif /* OAUTH_CURL_H */
diff --git a/src/interfaces/libpq-oauth/oauth-utils.c b/src/interfaces/libpq-oauth/oauth-utils.c
new file mode 100644
index 00000000000..57d543ac06f
--- /dev/null
+++ b/src/interfaces/libpq-oauth/oauth-utils.c
@@ -0,0 +1,233 @@
+/*-------------------------------------------------------------------------
+ *
+ * oauth-utils.c
+ *
+ * "Glue" helpers providing a copy of some internal APIs from libpq. At
+ * some point in the future, we might be able to deduplicate.
+ *
+ * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/interfaces/libpq-oauth/oauth-utils.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include <signal.h>
+
+#include "oauth-utils.h"
+
+#ifndef USE_DYNAMIC_OAUTH
+#error oauth-utils.c is not supported in static builds
+#endif
+
+#ifdef LIBPQ_INT_H
+#error do not rely on libpq-int.h in libpq-oauth
+#endif
+
+/*
+ * Function pointers set by libpq_oauth_init().
+ */
+
+pgthreadlock_t pg_g_threadlock;
+static libpq_gettext_func libpq_gettext_impl;
+
+conn_errorMessage_func conn_errorMessage;
+conn_oauth_client_id_func conn_oauth_client_id;
+conn_oauth_client_secret_func conn_oauth_client_secret;
+conn_oauth_discovery_uri_func conn_oauth_discovery_uri;
+conn_oauth_issuer_id_func conn_oauth_issuer_id;
+conn_oauth_scope_func conn_oauth_scope;
+conn_sasl_state_func conn_sasl_state;
+
+set_conn_altsock_func set_conn_altsock;
+set_conn_oauth_token_func set_conn_oauth_token;
+
+/*-
+ * Initializes libpq-oauth by setting necessary callbacks.
+ *
+ * The current implementation relies on the following private implementation
+ * details of libpq:
+ *
+ * - pg_g_threadlock: protects libcurl initialization if the underlying Curl
+ * installation is not threadsafe
+ *
+ * - libpq_gettext: translates error messages using libpq's message domain
+ *
+ * The implementation also needs access to several members of the PGconn struct,
+ * which are not guaranteed to stay in place across minor versions. Accessors
+ * (named conn_*) and mutators (named set_conn_*) are injected here.
+ */
+void
+libpq_oauth_init(pgthreadlock_t threadlock_impl,
+ libpq_gettext_func gettext_impl,
+ conn_errorMessage_func errmsg_impl,
+ conn_oauth_client_id_func clientid_impl,
+ conn_oauth_client_secret_func clientsecret_impl,
+ conn_oauth_discovery_uri_func discoveryuri_impl,
+ conn_oauth_issuer_id_func issuerid_impl,
+ conn_oauth_scope_func scope_impl,
+ conn_sasl_state_func saslstate_impl,
+ set_conn_altsock_func setaltsock_impl,
+ set_conn_oauth_token_func settoken_impl)
+{
+ pg_g_threadlock = threadlock_impl;
+ libpq_gettext_impl = gettext_impl;
+ conn_errorMessage = errmsg_impl;
+ conn_oauth_client_id = clientid_impl;
+ conn_oauth_client_secret = clientsecret_impl;
+ conn_oauth_discovery_uri = discoveryuri_impl;
+ conn_oauth_issuer_id = issuerid_impl;
+ conn_oauth_scope = scope_impl;
+ conn_sasl_state = saslstate_impl;
+ set_conn_altsock = setaltsock_impl;
+ set_conn_oauth_token = settoken_impl;
+}
+
+/*
+ * Append a formatted string to the error message buffer of the given
+ * connection, after translating it. This is a copy of libpq's internal API.
+ */
+void
+libpq_append_conn_error(PGconn *conn, const char *fmt,...)
+{
+ int save_errno = errno;
+ bool done;
+ va_list args;
+ PQExpBuffer errorMessage = conn_errorMessage(conn);
+
+ Assert(fmt[strlen(fmt) - 1] != '\n');
+
+ if (PQExpBufferBroken(errorMessage))
+ return; /* already failed */
+
+ /* Loop in case we have to retry after enlarging the buffer. */
+ do
+ {
+ errno = save_errno;
+ va_start(args, fmt);
+ done = appendPQExpBufferVA(errorMessage, libpq_gettext(fmt), args);
+ va_end(args);
+ } while (!done);
+
+ appendPQExpBufferChar(errorMessage, '\n');
+}
+
+#ifdef ENABLE_NLS
+
+/*
+ * A shim that defers to the actual libpq_gettext().
+ */
+char *
+libpq_gettext(const char *msgid)
+{
+ if (!libpq_gettext_impl)
+ {
+ /*
+ * Possible if the libpq build didn't enable NLS but the libpq-oauth
+ * build did. That's an odd mismatch, but we can handle it.
+ *
+ * Note that callers of libpq_gettext() have to treat the return value
+ * as if it were const, because builds without NLS simply pass through
+ * their argument.
+ */
+ return unconstify(char *, msgid);
+ }
+
+ return libpq_gettext_impl(msgid);
+}
+
+#endif /* ENABLE_NLS */
+
+/*
+ * Returns true if the PGOAUTHDEBUG=UNSAFE flag is set in the environment.
+ */
+bool
+oauth_unsafe_debugging_enabled(void)
+{
+ const char *env = getenv("PGOAUTHDEBUG");
+
+ return (env && strcmp(env, "UNSAFE") == 0);
+}
+
+/*
+ * Duplicate SOCK_ERRNO* definitions from libpq-int.h, for use by
+ * pq_block/reset_sigpipe().
+ */
+#ifdef WIN32
+#define SOCK_ERRNO (WSAGetLastError())
+#define SOCK_ERRNO_SET(e) WSASetLastError(e)
+#else
+#define SOCK_ERRNO errno
+#define SOCK_ERRNO_SET(e) (errno = (e))
+#endif
+
+/*
+ * Block SIGPIPE for this thread. This is a copy of libpq's internal API.
+ */
+int
+pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending)
+{
+ sigset_t sigpipe_sigset;
+ sigset_t sigset;
+
+ sigemptyset(&sigpipe_sigset);
+ sigaddset(&sigpipe_sigset, SIGPIPE);
+
+ /* Block SIGPIPE and save previous mask for later reset */
+ SOCK_ERRNO_SET(pthread_sigmask(SIG_BLOCK, &sigpipe_sigset, osigset));
+ if (SOCK_ERRNO)
+ return -1;
+
+ /* We can have a pending SIGPIPE only if it was blocked before */
+ if (sigismember(osigset, SIGPIPE))
+ {
+ /* Is there a pending SIGPIPE? */
+ if (sigpending(&sigset) != 0)
+ return -1;
+
+ if (sigismember(&sigset, SIGPIPE))
+ *sigpipe_pending = true;
+ else
+ *sigpipe_pending = false;
+ }
+ else
+ *sigpipe_pending = false;
+
+ return 0;
+}
+
+/*
+ * Discard any pending SIGPIPE and reset the signal mask. This is a copy of
+ * libpq's internal API.
+ */
+void
+pq_reset_sigpipe(sigset_t *osigset, bool sigpipe_pending, bool got_epipe)
+{
+ int save_errno = SOCK_ERRNO;
+ int signo;
+ sigset_t sigset;
+
+ /* Clear SIGPIPE only if none was pending */
+ if (got_epipe && !sigpipe_pending)
+ {
+ if (sigpending(&sigset) == 0 &&
+ sigismember(&sigset, SIGPIPE))
+ {
+ sigset_t sigpipe_sigset;
+
+ sigemptyset(&sigpipe_sigset);
+ sigaddset(&sigpipe_sigset, SIGPIPE);
+
+ sigwait(&sigpipe_sigset, &signo);
+ }
+ }
+
+ /* Restore saved block mask */
+ pthread_sigmask(SIG_SETMASK, osigset, NULL);
+
+ SOCK_ERRNO_SET(save_errno);
+}
diff --git a/src/interfaces/libpq-oauth/oauth-utils.h b/src/interfaces/libpq-oauth/oauth-utils.h
new file mode 100644
index 00000000000..f4ffefef208
--- /dev/null
+++ b/src/interfaces/libpq-oauth/oauth-utils.h
@@ -0,0 +1,94 @@
+/*-------------------------------------------------------------------------
+ *
+ * oauth-utils.h
+ *
+ * Definitions providing missing libpq internal APIs
+ *
+ * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/interfaces/libpq-oauth/oauth-utils.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef OAUTH_UTILS_H
+#define OAUTH_UTILS_H
+
+#include "fe-auth-oauth.h"
+#include "libpq-fe.h"
+#include "pqexpbuffer.h"
+
+/*
+ * A bank of callbacks to safely access members of PGconn, which are all passed
+ * to libpq_oauth_init() by libpq.
+ *
+ * Keep these aligned with the definitions in fe-auth-oauth.c as well as the
+ * static declarations in oauth-curl.c.
+ */
+#define DECLARE_GETTER(TYPE, MEMBER) \
+ typedef TYPE (*conn_ ## MEMBER ## _func) (PGconn *conn); \
+ extern conn_ ## MEMBER ## _func conn_ ## MEMBER;
+
+#define DECLARE_SETTER(TYPE, MEMBER) \
+ typedef void (*set_conn_ ## MEMBER ## _func) (PGconn *conn, TYPE val); \
+ extern set_conn_ ## MEMBER ## _func set_conn_ ## MEMBER;
+
+DECLARE_GETTER(PQExpBuffer, errorMessage);
+DECLARE_GETTER(char *, oauth_client_id);
+DECLARE_GETTER(char *, oauth_client_secret);
+DECLARE_GETTER(char *, oauth_discovery_uri);
+DECLARE_GETTER(char *, oauth_issuer_id);
+DECLARE_GETTER(char *, oauth_scope);
+DECLARE_GETTER(fe_oauth_state *, sasl_state);
+
+DECLARE_SETTER(pgsocket, altsock);
+DECLARE_SETTER(char *, oauth_token);
+
+#undef DECLARE_GETTER
+#undef DECLARE_SETTER
+
+typedef char *(*libpq_gettext_func) (const char *msgid);
+
+/* Initializes libpq-oauth. */
+extern PGDLLEXPORT void libpq_oauth_init(pgthreadlock_t threadlock,
+ libpq_gettext_func gettext_impl,
+ conn_errorMessage_func errmsg_impl,
+ conn_oauth_client_id_func clientid_impl,
+ conn_oauth_client_secret_func clientsecret_impl,
+ conn_oauth_discovery_uri_func discoveryuri_impl,
+ conn_oauth_issuer_id_func issuerid_impl,
+ conn_oauth_scope_func scope_impl,
+ conn_sasl_state_func saslstate_impl,
+ set_conn_altsock_func setaltsock_impl,
+ set_conn_oauth_token_func settoken_impl);
+
+/*
+ * Duplicated APIs, copied from libpq (primarily libpq-int.h, which we cannot
+ * depend on here).
+ */
+
+typedef enum
+{
+ PG_BOOL_UNKNOWN = 0, /* Currently unknown */
+ PG_BOOL_YES, /* Yes (true) */
+ PG_BOOL_NO /* No (false) */
+} PGTernaryBool;
+
+extern void libpq_append_conn_error(PGconn *conn, const char *fmt,...) pg_attribute_printf(2, 3);
+extern bool oauth_unsafe_debugging_enabled(void);
+extern int pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending);
+extern void pq_reset_sigpipe(sigset_t *osigset, bool sigpipe_pending, bool got_epipe);
+
+#ifdef ENABLE_NLS
+extern char *libpq_gettext(const char *msgid) pg_attribute_format_arg(1);
+#else
+#define libpq_gettext(x) (x)
+#endif
+
+extern pgthreadlock_t pg_g_threadlock;
+
+#define pglock_thread() pg_g_threadlock(true)
+#define pgunlock_thread() pg_g_threadlock(false)
+
+#endif /* OAUTH_UTILS_H */
diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile
index 90b0b65db6f..c6fe5fec7f6 100644
--- a/src/interfaces/libpq/Makefile
+++ b/src/interfaces/libpq/Makefile
@@ -31,7 +31,6 @@ endif
OBJS = \
$(WIN32RES) \
- fe-auth-oauth.o \
fe-auth-scram.o \
fe-cancel.o \
fe-connect.o \
@@ -64,9 +63,11 @@ OBJS += \
fe-secure-gssapi.o
endif
-ifeq ($(with_libcurl),yes)
-OBJS += fe-auth-oauth-curl.o
-endif
+# The OAuth implementation differs depending on the type of library being built.
+OBJS_STATIC = fe-auth-oauth.o
+
+fe-auth-oauth_shlib.o: override CPPFLAGS_SHLIB += -DUSE_DYNAMIC_OAUTH
+OBJS_SHLIB = fe-auth-oauth_shlib.o
ifeq ($(PORTNAME), cygwin)
override shlib = cyg$(NAME)$(DLSUFFIX)
@@ -86,7 +87,7 @@ endif
# that are built correctly for use in a shlib.
SHLIB_LINK_INTERNAL = -lpgcommon_shlib -lpgport_shlib
ifneq ($(PORTNAME), win32)
-SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lcurl -lsocket -lnsl -lresolv -lintl -lm, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS)
+SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS)
else
SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE)
endif
@@ -101,12 +102,26 @@ ifeq ($(with_ssl),openssl)
PKG_CONFIG_REQUIRES_PRIVATE = libssl, libcrypto
endif
+ifeq ($(with_libcurl),yes)
+# libpq.so doesn't link against libcurl, but libpq.a needs libpq-oauth, and
+# libpq-oauth needs libcurl. Put both into *.private.
+PKG_CONFIG_REQUIRES_PRIVATE += libcurl
+%.pc: override SHLIB_LINK_INTERNAL += -lpq-oauth
+endif
+
all: all-lib libpq-refs-stamp
# Shared library stuff
include $(top_srcdir)/src/Makefile.shlib
backend_src = $(top_srcdir)/src/backend
+# Add shlib-/stlib-specific objects.
+$(shlib): override OBJS += $(OBJS_SHLIB)
+$(shlib): $(OBJS_SHLIB)
+
+$(stlib): override OBJS += $(OBJS_STATIC)
+$(stlib): $(OBJS_STATIC)
+
# Check for functions that libpq must not call, currently just exit().
# (Ideally we'd reject abort() too, but there are various scenarios where
# build toolchains insert abort() calls, e.g. to implement assert().)
@@ -115,8 +130,6 @@ backend_src = $(top_srcdir)/src/backend
# which seems to insert references to that even in pure C code. Excluding
# __tsan_func_exit is necessary when using ThreadSanitizer data race detector
# which use this function for instrumentation of function exit.
-# libcurl registers an exit handler in the memory debugging code when running
-# with LeakSanitizer.
# Skip the test when profiling, as gcc may insert exit() calls for that.
# Also skip the test on platforms where libpq infrastructure may be provided
# by statically-linked libraries, as we can't expect them to honor this
@@ -124,7 +137,7 @@ backend_src = $(top_srcdir)/src/backend
libpq-refs-stamp: $(shlib)
ifneq ($(enable_coverage), yes)
ifeq (,$(filter solaris,$(PORTNAME)))
- @if nm -A -u $< 2>/dev/null | grep -v -e __cxa_atexit -e __tsan_func_exit -e _atexit | grep exit; then \
+ @if nm -A -u $< 2>/dev/null | grep -v -e __cxa_atexit -e __tsan_func_exit | grep exit; then \
echo 'libpq must not be calling any function which invokes exit'; exit 1; \
fi
endif
@@ -138,6 +151,11 @@ fe-misc.o: fe-misc.c $(top_builddir)/src/port/pg_config_paths.h
$(top_builddir)/src/port/pg_config_paths.h:
$(MAKE) -C $(top_builddir)/src/port pg_config_paths.h
+# Use src/common/Makefile's trick for tracking dependencies of shlib-specific
+# objects.
+%_shlib.o: %.c %.o
+ $(CC) $(CFLAGS) $(CFLAGS_SL) $(CPPFLAGS) $(CPPFLAGS_SHLIB) -c $< -o $@
+
install: all installdirs install-lib
$(INSTALL_DATA) $(srcdir)/libpq-fe.h '$(DESTDIR)$(includedir)'
$(INSTALL_DATA) $(srcdir)/libpq-events.h '$(DESTDIR)$(includedir)'
@@ -171,6 +189,6 @@ uninstall: uninstall-lib
clean distclean: clean-lib
$(MAKE) -C test $@
rm -rf tmp_check
- rm -f $(OBJS) pthread.h libpq-refs-stamp
+ rm -f $(OBJS) $(OBJS_SHLIB) $(OBJS_STATIC) pthread.h libpq-refs-stamp
# Might be left over from a Win32 client-only build
rm -f pg_config_paths.h
diff --git a/src/interfaces/libpq/exports.txt b/src/interfaces/libpq/exports.txt
index d5143766858..0625cf39e9a 100644
--- a/src/interfaces/libpq/exports.txt
+++ b/src/interfaces/libpq/exports.txt
@@ -210,3 +210,4 @@ PQsetAuthDataHook 207
PQgetAuthDataHook 208
PQdefaultAuthDataHook 209
PQfullProtocolVersion 210
+appendPQExpBufferVA 211
diff --git a/src/interfaces/libpq/fe-auth-oauth.c b/src/interfaces/libpq/fe-auth-oauth.c
index ab6a45e2aba..9fbff89a21d 100644
--- a/src/interfaces/libpq/fe-auth-oauth.c
+++ b/src/interfaces/libpq/fe-auth-oauth.c
@@ -15,6 +15,10 @@
#include "postgres_fe.h"
+#ifdef USE_DYNAMIC_OAUTH
+#include <dlfcn.h>
+#endif
+
#include "common/base64.h"
#include "common/hmac.h"
#include "common/jsonapi.h"
@@ -22,6 +26,7 @@
#include "fe-auth.h"
#include "fe-auth-oauth.h"
#include "mb/pg_wchar.h"
+#include "pg_config_paths.h"
/* The exported OAuth callback mechanism. */
static void *oauth_init(PGconn *conn, const char *password,
@@ -721,6 +726,218 @@ cleanup_user_oauth_flow(PGconn *conn)
state->async_ctx = NULL;
}
+/*-------------
+ * Builtin Flow
+ *
+ * There are three potential implementations of use_builtin_flow:
+ *
+ * 1) If the OAuth client is disabled at configuration time, return false.
+ * Dependent clients must provide their own flow.
+ * 2) If the OAuth client is enabled and USE_DYNAMIC_OAUTH is defined, dlopen()
+ * the libpq-oauth plugin and use its implementation.
+ * 3) Otherwise, use flow callbacks that are statically linked into the
+ * executable.
+ */
+
+#if !defined(USE_LIBCURL)
+
+/*
+ * This configuration doesn't support the builtin flow.
+ */
+
+bool
+use_builtin_flow(PGconn *conn, fe_oauth_state *state)
+{
+ return false;
+}
+
+#elif defined(USE_DYNAMIC_OAUTH)
+
+/*
+ * Use the builtin flow in the libpq-oauth plugin, which is loaded at runtime.
+ */
+
+typedef char *(*libpq_gettext_func) (const char *msgid);
+
+/*
+ * Define accessor/mutator shims to inject into libpq-oauth, so that it doesn't
+ * depend on the offsets within PGconn. (These have changed during minor version
+ * updates in the past.)
+ */
+
+#define DEFINE_GETTER(TYPE, MEMBER) \
+ typedef TYPE (*conn_ ## MEMBER ## _func) (PGconn *conn); \
+ static TYPE conn_ ## MEMBER(PGconn *conn) { return conn->MEMBER; }
+
+/* Like DEFINE_GETTER, but returns a pointer to the member. */
+#define DEFINE_GETTER_P(TYPE, MEMBER) \
+ typedef TYPE (*conn_ ## MEMBER ## _func) (PGconn *conn); \
+ static TYPE conn_ ## MEMBER(PGconn *conn) { return &conn->MEMBER; }
+
+#define DEFINE_SETTER(TYPE, MEMBER) \
+ typedef void (*set_conn_ ## MEMBER ## _func) (PGconn *conn, TYPE val); \
+ static void set_conn_ ## MEMBER(PGconn *conn, TYPE val) { conn->MEMBER = val; }
+
+DEFINE_GETTER_P(PQExpBuffer, errorMessage);
+DEFINE_GETTER(char *, oauth_client_id);
+DEFINE_GETTER(char *, oauth_client_secret);
+DEFINE_GETTER(char *, oauth_discovery_uri);
+DEFINE_GETTER(char *, oauth_issuer_id);
+DEFINE_GETTER(char *, oauth_scope);
+DEFINE_GETTER(fe_oauth_state *, sasl_state);
+
+DEFINE_SETTER(pgsocket, altsock);
+DEFINE_SETTER(char *, oauth_token);
+
+/*
+ * Loads the libpq-oauth plugin via dlopen(), initializes it, and plugs its
+ * callbacks into the connection's async auth handlers.
+ *
+ * Failure to load here results in a relatively quiet connection error, to
+ * handle the use case where the build supports loading a flow but a user does
+ * not want to install it. Troubleshooting of linker/loader failures can be done
+ * via PGOAUTHDEBUG.
+ */
+bool
+use_builtin_flow(PGconn *conn, fe_oauth_state *state)
+{
+ static bool initialized = false;
+ static pthread_mutex_t init_mutex = PTHREAD_MUTEX_INITIALIZER;
+ int lockerr;
+
+ void (*init) (pgthreadlock_t threadlock,
+ libpq_gettext_func gettext_impl,
+ conn_errorMessage_func errmsg_impl,
+ conn_oauth_client_id_func clientid_impl,
+ conn_oauth_client_secret_func clientsecret_impl,
+ conn_oauth_discovery_uri_func discoveryuri_impl,
+ conn_oauth_issuer_id_func issuerid_impl,
+ conn_oauth_scope_func scope_impl,
+ conn_sasl_state_func saslstate_impl,
+ set_conn_altsock_func setaltsock_impl,
+ set_conn_oauth_token_func settoken_impl);
+ PostgresPollingStatusType (*flow) (PGconn *conn);
+ void (*cleanup) (PGconn *conn);
+
+ /*
+ * On macOS only, load the module using its absolute install path; the
+ * standard search behavior is not very helpful for this use case. Unlike
+ * on other platforms, DYLD_LIBRARY_PATH is used as a fallback even with
+ * absolute paths (modulo SIP effects), so tests can continue to work.
+ *
+ * On the other platforms, load the module using only the basename, to
+ * rely on the runtime linker's standard search behavior.
+ */
+ const char *const module_name =
+#if defined(__darwin__)
+ LIBDIR "/libpq-oauth-" PG_MAJORVERSION DLSUFFIX;
+#else
+ "libpq-oauth-" PG_MAJORVERSION DLSUFFIX;
+#endif
+
+ state->builtin_flow = dlopen(module_name, RTLD_NOW | RTLD_LOCAL);
+ if (!state->builtin_flow)
+ {
+ /*
+ * For end users, this probably isn't an error condition, it just
+ * means the flow isn't installed. Developers and package maintainers
+ * may want to debug this via the PGOAUTHDEBUG envvar, though.
+ *
+ * Note that POSIX dlerror() isn't guaranteed to be threadsafe.
+ */
+ if (oauth_unsafe_debugging_enabled())
+ fprintf(stderr, "failed dlopen for libpq-oauth: %s\n", dlerror());
+
+ return false;
+ }
+
+ if ((init = dlsym(state->builtin_flow, "libpq_oauth_init")) == NULL
+ || (flow = dlsym(state->builtin_flow, "pg_fe_run_oauth_flow")) == NULL
+ || (cleanup = dlsym(state->builtin_flow, "pg_fe_cleanup_oauth_flow")) == NULL)
+ {
+ /*
+ * This is more of an error condition than the one above, but due to
+ * the dlerror() threadsafety issue, lock it behind PGOAUTHDEBUG too.
+ */
+ if (oauth_unsafe_debugging_enabled())
+ fprintf(stderr, "failed dlsym for libpq-oauth: %s\n", dlerror());
+
+ dlclose(state->builtin_flow);
+ return false;
+ }
+
+ /*
+ * Past this point, we do not unload the module. It stays in the process
+ * permanently.
+ */
+
+ /*
+ * We need to inject necessary function pointers into the module. This
+ * only needs to be done once -- even if the pointers are constant,
+ * assigning them while another thread is executing the flows feels like
+ * tempting fate.
+ */
+ if ((lockerr = pthread_mutex_lock(&init_mutex)) != 0)
+ {
+ /* Should not happen... but don't continue if it does. */
+ Assert(false);
+
+ libpq_append_conn_error(conn, "failed to lock mutex (%d)", lockerr);
+ return false;
+ }
+
+ if (!initialized)
+ {
+ init(pg_g_threadlock,
+#ifdef ENABLE_NLS
+ libpq_gettext,
+#else
+ NULL,
+#endif
+ conn_errorMessage,
+ conn_oauth_client_id,
+ conn_oauth_client_secret,
+ conn_oauth_discovery_uri,
+ conn_oauth_issuer_id,
+ conn_oauth_scope,
+ conn_sasl_state,
+ set_conn_altsock,
+ set_conn_oauth_token);
+
+ initialized = true;
+ }
+
+ pthread_mutex_unlock(&init_mutex);
+
+ /* Set our asynchronous callbacks. */
+ conn->async_auth = flow;
+ conn->cleanup_async_auth = cleanup;
+
+ return true;
+}
+
+#else
+
+/*
+ * Use the builtin flow in libpq-oauth.a (see libpq-oauth/oauth-curl.h).
+ */
+
+extern PostgresPollingStatusType pg_fe_run_oauth_flow(PGconn *conn);
+extern void pg_fe_cleanup_oauth_flow(PGconn *conn);
+
+bool
+use_builtin_flow(PGconn *conn, fe_oauth_state *state)
+{
+ /* Set our asynchronous callbacks. */
+ conn->async_auth = pg_fe_run_oauth_flow;
+ conn->cleanup_async_auth = pg_fe_cleanup_oauth_flow;
+
+ return true;
+}
+
+#endif /* USE_LIBCURL */
+
+
/*
* Chooses an OAuth client flow for the connection, which will retrieve a Bearer
* token for presentation to the server.
@@ -792,18 +1009,10 @@ setup_token_request(PGconn *conn, fe_oauth_state *state)
libpq_append_conn_error(conn, "user-defined OAuth flow failed");
goto fail;
}
- else
+ else if (!use_builtin_flow(conn, state))
{
-#if USE_LIBCURL
- /* Hand off to our built-in OAuth flow. */
- conn->async_auth = pg_fe_run_oauth_flow;
- conn->cleanup_async_auth = pg_fe_cleanup_oauth_flow;
-
-#else
- libpq_append_conn_error(conn, "no custom OAuth flows are available, and libpq was not built with libcurl support");
+ libpq_append_conn_error(conn, "no OAuth flows are available (try installing the libpq-oauth package)");
goto fail;
-
-#endif
}
return true;
diff --git a/src/interfaces/libpq/fe-auth-oauth.h b/src/interfaces/libpq/fe-auth-oauth.h
index 3f1a7503a01..0d59e91605b 100644
--- a/src/interfaces/libpq/fe-auth-oauth.h
+++ b/src/interfaces/libpq/fe-auth-oauth.h
@@ -15,8 +15,8 @@
#ifndef FE_AUTH_OAUTH_H
#define FE_AUTH_OAUTH_H
+#include "fe-auth-sasl.h"
#include "libpq-fe.h"
-#include "libpq-int.h"
enum fe_oauth_step
@@ -27,18 +27,24 @@ enum fe_oauth_step
FE_OAUTH_SERVER_ERROR,
};
+/*
+ * This struct is exported to the libpq-oauth module. If changes are needed
+ * during backports to stable branches, please keep ABI compatibility (no
+ * changes to existing members, add new members at the end, etc.).
+ */
typedef struct
{
enum fe_oauth_step step;
PGconn *conn;
void *async_ctx;
+
+ void *builtin_flow;
} fe_oauth_state;
-extern PostgresPollingStatusType pg_fe_run_oauth_flow(PGconn *conn);
-extern void pg_fe_cleanup_oauth_flow(PGconn *conn);
extern void pqClearOAuthToken(PGconn *conn);
extern bool oauth_unsafe_debugging_enabled(void);
+extern bool use_builtin_flow(PGconn *conn, fe_oauth_state *state);
/* Mechanisms in fe-auth-oauth.c */
extern const pg_fe_sasl_mech pg_oauth_mech;
diff --git a/src/interfaces/libpq/meson.build b/src/interfaces/libpq/meson.build
index 292fecf3320..a74e885b169 100644
--- a/src/interfaces/libpq/meson.build
+++ b/src/interfaces/libpq/meson.build
@@ -38,10 +38,6 @@ if gssapi.found()
)
endif
-if libcurl.found()
- libpq_sources += files('fe-auth-oauth-curl.c')
-endif
-
export_file = custom_target('libpq.exports',
kwargs: gen_export_kwargs,
)
@@ -50,6 +46,9 @@ export_file = custom_target('libpq.exports',
libpq_inc = include_directories('.', '../../port')
libpq_c_args = ['-DSO_MAJOR_VERSION=5']
+# The OAuth implementation differs depending on the type of library being built.
+libpq_so_c_args = ['-DUSE_DYNAMIC_OAUTH']
+
# Not using both_libraries() here as
# 1) resource files should only be in the shared library
# 2) we want the .pc file to include a dependency to {pgport,common}_static for
@@ -70,7 +69,7 @@ libpq_st = static_library('libpq',
libpq_so = shared_library('libpq',
libpq_sources + libpq_so_sources,
include_directories: [libpq_inc, postgres_inc],
- c_args: libpq_c_args,
+ c_args: libpq_c_args + libpq_so_c_args,
c_pch: pch_postgres_fe_h,
version: '5.' + pg_version_major.to_string(),
soversion: host_system != 'windows' ? '5' : '',
@@ -86,12 +85,26 @@ libpq = declare_dependency(
include_directories: [include_directories('.')]
)
+private_deps = [
+ frontend_stlib_code,
+ libpq_deps,
+]
+
+if oauth_flow_supported
+ # libpq.so doesn't link against libcurl, but libpq.a needs libpq-oauth, and
+ # libpq-oauth needs libcurl. Put both into *.private.
+ private_deps += [
+ libpq_oauth_deps,
+ '-lpq-oauth',
+ ]
+endif
+
pkgconfig.generate(
name: 'libpq',
description: 'PostgreSQL libpq library',
url: pg_url,
libraries: libpq,
- libraries_private: [frontend_stlib_code, libpq_deps],
+ libraries_private: private_deps,
)
install_headers(
diff --git a/src/interfaces/libpq/nls.mk b/src/interfaces/libpq/nls.mk
index ae761265852..b87df277d93 100644
--- a/src/interfaces/libpq/nls.mk
+++ b/src/interfaces/libpq/nls.mk
@@ -13,15 +13,21 @@ GETTEXT_FILES = fe-auth.c \
fe-secure-common.c \
fe-secure-gssapi.c \
fe-secure-openssl.c \
- win32.c
-GETTEXT_TRIGGERS = libpq_append_conn_error:2 \
+ win32.c \
+ ../libpq-oauth/oauth-curl.c \
+ ../libpq-oauth/oauth-utils.c
+GETTEXT_TRIGGERS = actx_error:2 \
+ libpq_append_conn_error:2 \
libpq_append_error:2 \
libpq_gettext \
libpq_ngettext:1,2 \
+ oauth_parse_set_error:2 \
pqInternalNotice:2
-GETTEXT_FLAGS = libpq_append_conn_error:2:c-format \
+GETTEXT_FLAGS = actx_error:2:c-format \
+ libpq_append_conn_error:2:c-format \
libpq_append_error:2:c-format \
libpq_gettext:1:pass-c-format \
libpq_ngettext:1:pass-c-format \
libpq_ngettext:2:pass-c-format \
+ oauth_parse_set_error:2:c-format \
pqInternalNotice:2:c-format
diff --git a/src/makefiles/meson.build b/src/makefiles/meson.build
index 55da678ec27..91a8de1ee9b 100644
--- a/src/makefiles/meson.build
+++ b/src/makefiles/meson.build
@@ -203,6 +203,8 @@ pgxs_empty = [
'LIBNUMA_CFLAGS', 'LIBNUMA_LIBS',
'LIBURING_CFLAGS', 'LIBURING_LIBS',
+
+ 'LIBCURL_CPPFLAGS', 'LIBCURL_LDFLAGS', 'LIBCURL_LDLIBS',
]
if host_system == 'windows' and cc.get_argument_syntax() != 'msvc'
diff --git a/src/test/modules/oauth_validator/meson.build b/src/test/modules/oauth_validator/meson.build
index 36d1b26369f..e190f9cf15a 100644
--- a/src/test/modules/oauth_validator/meson.build
+++ b/src/test/modules/oauth_validator/meson.build
@@ -78,7 +78,7 @@ tests += {
],
'env': {
'PYTHON': python.path(),
- 'with_libcurl': libcurl.found() ? 'yes' : 'no',
+ 'with_libcurl': oauth_flow_supported ? 'yes' : 'no',
'with_python': 'yes',
},
},
diff --git a/src/test/modules/oauth_validator/t/002_client.pl b/src/test/modules/oauth_validator/t/002_client.pl
index 8dd502f41e1..21d4acc1926 100644
--- a/src/test/modules/oauth_validator/t/002_client.pl
+++ b/src/test/modules/oauth_validator/t/002_client.pl
@@ -110,7 +110,7 @@ if ($ENV{with_libcurl} ne 'yes')
"fails without custom hook installed",
flags => ["--no-hook"],
expected_stderr =>
- qr/no custom OAuth flows are available, and libpq was not built with libcurl support/
+ qr/no OAuth flows are available \(try installing the libpq-oauth package\)/
);
}
--
2.34.1
^ permalink raw reply [nested|flat] 6+ messages in thread
end of thread, other threads:[~2025-04-29 00:10 UTC | newest]
Thread overview: 6+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-04-02 00:57 [PATCH 4/8] BRIN bloom indexes Tomas Vondra <[email protected]>
2025-04-23 15:39 Re: [PoC] Federated Authn/z with OAUTHBEARER Christoph Berg <[email protected]>
2025-04-23 16:07 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Jacob Champion <[email protected]>
2025-04-23 16:38 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Christoph Berg <[email protected]>
2025-04-23 17:46 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Jacob Champion <[email protected]>
2025-04-29 00:10 ` Re: [PoC] Federated Authn/z with OAUTHBEARER Jacob Champion <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox