public inbox for [email protected]
help / color / mirror / Atom feedBug in comparison of empty jsonb arrays to scalars
31+ messages / 7 participants
[nested] [flat]
* Bug in comparison of empty jsonb arrays to scalars
@ 2016-11-08 23:31 Nikita Glukhov <[email protected]>
2016-11-09 02:49 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
0 siblings, 1 reply; 31+ messages in thread
From: Nikita Glukhov @ 2016-11-08 23:31 UTC (permalink / raw)
To: pgsql-hackers
Hi hackers.
While working on jsonbstatistics, I found the following bug:
an empty jsonb array is considered to be lesser than any scalar,
but it is expected that objects > arrays > scalars.
# select '[]'::jsonb < 'null'::jsonb;
?column?
----------
t
(1 row)
Attached patch contains:
1. bug fix (added the missing "else" in compareJsonbContainers())
2. regression test
3. pg_upgrade: invalidation of btree indexes on jsonb columns and
REINDEX-script generation
--
Nikita Glukhov
Postgres Professional: http://www.postgrespro.com
The Russian Postgres Company
--
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Attachments:
[text/x-patch] fix_empty_jsonb_array_comparison_v01.patch (8.0K, ../../[email protected]/2-fix_empty_jsonb_array_comparison_v01.patch)
download | inline diff:
diff --git a/src/backend/utils/adt/jsonb_util.c b/src/backend/utils/adt/jsonb_util.c
index ddc34ce..43934bf 100644
--- a/src/backend/utils/adt/jsonb_util.c
+++ b/src/backend/utils/adt/jsonb_util.c
@@ -232,7 +232,7 @@ compareJsonbContainers(JsonbContainer *a, JsonbContainer *b)
*/
if (va.val.array.rawScalar != vb.val.array.rawScalar)
res = (va.val.array.rawScalar) ? -1 : 1;
- if (va.val.array.nElems != vb.val.array.nElems)
+ else if (va.val.array.nElems != vb.val.array.nElems)
res = (va.val.array.nElems > vb.val.array.nElems) ? 1 : -1;
break;
case jbvObject:
diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c
index 42bf499..81c1616 100644
--- a/src/bin/pg_upgrade/check.c
+++ b/src/bin/pg_upgrade/check.c
@@ -115,6 +115,11 @@ check_and_dump_old_cluster(bool live_check)
if (GET_MAJOR_VERSION(old_cluster.major_version) <= 804)
new_9_0_populate_pg_largeobject_metadata(&old_cluster, true);
+ /* Pre-PG 10.0 had bug in jsonb comparison operator */
+ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 906 &&
+ GET_MAJOR_VERSION(old_cluster.major_version) >= 904)
+ old_9_6_invalidate_jsonb_btree_indexes(&old_cluster, true);
+
/*
* While not a check option, we do this now because this is the only time
* the old server is running.
@@ -166,11 +171,26 @@ report_clusters_compatible(void)
void
issue_warnings(void)
{
- /* Create dummy large object permissions for old < PG 9.0? */
- if (GET_MAJOR_VERSION(old_cluster.major_version) <= 804)
+ bool need_new_9_0_populate_pg_largeobject_metadata =
+ GET_MAJOR_VERSION(old_cluster.major_version) <= 804;
+
+ bool need_old_9_6_invalidate_jsonb_btree_indexes =
+ GET_MAJOR_VERSION(old_cluster.major_version) <= 906 &&
+ GET_MAJOR_VERSION(old_cluster.major_version) >= 904;
+
+ if (need_new_9_0_populate_pg_largeobject_metadata ||
+ need_old_9_6_invalidate_jsonb_btree_indexes)
{
start_postmaster(&new_cluster, true);
- new_9_0_populate_pg_largeobject_metadata(&new_cluster, false);
+
+ /* Create dummy large object permissions for old < PG 9.0? */
+ if (need_new_9_0_populate_pg_largeobject_metadata)
+ new_9_0_populate_pg_largeobject_metadata(&new_cluster, false);
+
+ /* invalidate jsonb btree indexes for old < PG 10.0 */
+ if (need_old_9_6_invalidate_jsonb_btree_indexes)
+ old_9_6_invalidate_jsonb_btree_indexes(&new_cluster, false);
+
stop_postmaster(false);
}
}
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 19dca83..07e0ca6 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -442,6 +442,8 @@ void pg_putenv(const char *var, const char *val);
void new_9_0_populate_pg_largeobject_metadata(ClusterInfo *cluster,
bool check_mode);
void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster);
+void old_9_6_invalidate_jsonb_btree_indexes(ClusterInfo *cluster,
+ bool check_mode);
/* parallel.c */
void parallel_exec_prog(const char *log_file, const char *opt_log_file,
diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c
index 3c7c5fa..b1a3b89 100644
--- a/src/bin/pg_upgrade/version.c
+++ b/src/bin/pg_upgrade/version.c
@@ -10,6 +10,7 @@
#include "postgres_fe.h"
#include "pg_upgrade.h"
+#include "catalog/pg_type.h"
#include "fe_utils/string_utils.h"
@@ -185,3 +186,116 @@ old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster)
else
check_ok();
}
+
+/*
+ * old_9_6_invalidate_jsonb_btree_indexes()
+ * 9.4-9.6 -> 10.0
+ * Btree index ordering for jsonb had been fixed in 10.0
+ */
+void
+old_9_6_invalidate_jsonb_btree_indexes(ClusterInfo *cluster, bool check_mode)
+{
+ int dbnum;
+ FILE *script = NULL;
+ bool found = false;
+ char output_path[MAXPGPATH];
+
+ prep_status("Checking for jsonb btree indexes existence");
+
+ snprintf(output_path, sizeof(output_path), "reindex_jsonb_btree.sql");
+
+ for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++)
+ {
+ PGresult *res;
+ bool db_used = false;
+ int ntups;
+ int rowno;
+ int i_nspname,
+ i_relname;
+ DbInfo *active_db = &cluster->dbarr.dbs[dbnum];
+ PGconn *conn = connectToServer(cluster, active_db->db_name);
+
+ /* find jsonb btree indexes */
+ res = executeQueryOrDie(conn,
+ "SELECT DISTINCT n.nspname, c.relname "
+ "FROM pg_catalog.pg_class c, "
+ " pg_catalog.pg_index i, "
+ " pg_catalog.pg_am am, "
+ " pg_catalog.pg_namespace n, "
+ " pg_catalog.pg_attribute a "
+ "WHERE i.indexrelid = c.oid AND "
+ " c.relam = am.oid AND "
+ " c.relnamespace = n.oid AND "
+ " a.attrelid = c.oid AND "
+ " a.atttypid = " CppAsString2(JSONBOID) " AND "
+ " am.amname = 'btree'");
+
+ ntups = PQntuples(res);
+ i_nspname = PQfnumber(res, "nspname");
+ i_relname = PQfnumber(res, "relname");
+
+ for (rowno = 0; rowno < ntups; rowno++)
+ {
+ found = true;
+ if (!check_mode)
+ {
+ if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL)
+ pg_fatal("could not open file \"%s\": %s\n", output_path, strerror(errno));
+ if (!db_used)
+ {
+ fprintf(script, "\\connect %s\n",
+ quote_identifier(active_db->db_name));
+ db_used = true;
+ }
+ fprintf(script, "REINDEX INDEX %s.%s;\n",
+ quote_identifier(PQgetvalue(res, rowno, i_nspname)),
+ quote_identifier(PQgetvalue(res, rowno, i_relname)));
+ }
+ }
+
+ PQclear(res);
+
+ if (!check_mode && found)
+ /* mark jsonb btree indexes as invalid */
+ PQclear(executeQueryOrDie(conn,
+ "UPDATE pg_catalog.pg_index i "
+ "SET indisvalid = false "
+ "FROM pg_catalog.pg_class c, "
+ " pg_catalog.pg_am am, "
+ " pg_catalog.pg_namespace n, "
+ " pg_catalog.pg_attribute a "
+ "WHERE i.indexrelid = c.oid AND "
+ " c.relam = am.oid AND "
+ " c.relnamespace = n.oid AND "
+ " a.attrelid = c.oid AND "
+ " a.atttypid = " CppAsString2(JSONBOID) " AND "
+ " am.amname = 'btree'"));
+
+ PQfinish(conn);
+ }
+
+ if (script)
+ fclose(script);
+
+ if (found)
+ {
+ report_status(PG_WARNING, "warning");
+ if (check_mode)
+ pg_log(PG_WARNING, "\n"
+ "Your installation contains jsonb btree indexes. These indexes have\n"
+ "different comparison rules between your old and new clusters, so they\n"
+ "must be reindexed with the REINDEX command. After upgrading, you will\n"
+ "be given REINDEX instructions.\n\n");
+ else
+ pg_log(PG_WARNING, "\n"
+ "Your installation contains jsonb btree indexes. These indexes have\n"
+ "different comparison rules between your old and new clusters, so they\n"
+ "must be reindexed with the REINDEX command. The file:\n"
+ " %s\n"
+ "when executed by psql by the database superuser will recreate all invalid\n"
+ "indexes; until then, none of these indexes will be used.\n\n",
+ output_path);
+ }
+ else
+ check_ok();
+}
diff --git a/src/test/regress/expected/jsonb.out b/src/test/regress/expected/jsonb.out
index e2cb08a..203088d 100644
--- a/src/test/regress/expected/jsonb.out
+++ b/src/test/regress/expected/jsonb.out
@@ -2256,6 +2256,13 @@ SELECT count(*) FROM testjsonb WHERE j = '{"pos":98, "line":371, "node":"CBA", "
1
(1 row)
+-- bug in comparison of [] and scalars
+SELECT '[]'::jsonb > 'null'::jsonb;
+ ?column?
+----------
+ t
+(1 row)
+
--gin path opclass
DROP INDEX jidx;
CREATE INDEX jidx ON testjsonb USING gin (j jsonb_path_ops);
diff --git a/src/test/regress/sql/jsonb.sql b/src/test/regress/sql/jsonb.sql
index 6b4c796..2d37c4f 100644
--- a/src/test/regress/sql/jsonb.sql
+++ b/src/test/regress/sql/jsonb.sql
@@ -580,6 +580,9 @@ SET enable_seqscan = off;
SELECT count(*) FROM testjsonb WHERE j > '{"p":1}';
SELECT count(*) FROM testjsonb WHERE j = '{"pos":98, "line":371, "node":"CBA", "indexed":true}';
+-- bug in comparison of [] and scalars
+SELECT '[]'::jsonb > 'null'::jsonb;
+
--gin path opclass
DROP INDEX jidx;
CREATE INDEX jidx ON testjsonb USING gin (j jsonb_path_ops);
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Bug in comparison of empty jsonb arrays to scalars
2016-11-08 23:31 Bug in comparison of empty jsonb arrays to scalars Nikita Glukhov <[email protected]>
@ 2016-11-09 02:49 ` Michael Paquier <[email protected]>
2016-11-09 18:27 ` Re: Bug in comparison of empty jsonb arrays to scalars Robert Haas <[email protected]>
0 siblings, 1 reply; 31+ messages in thread
From: Michael Paquier @ 2016-11-09 02:49 UTC (permalink / raw)
To: Nikita Glukhov <[email protected]>; +Cc: pgsql-hackers
On Wed, Nov 9, 2016 at 8:31 AM, Nikita Glukhov <[email protected]> wrote:
> Hi hackers.
>
> While working on jsonbstatistics, I found the following bug:
> an empty jsonb array is considered to be lesser than any scalar,
> but it is expected that objects > arrays > scalars.
Sources? Does the JSON spec contain any information regarding
comparison operators? I don't think so, so that would be up to the
implementation to decide that, no?
Btw I would agree with you that's quite unintuitive, but that's not
wrong either to keep the current comparison algorithm because that's
harmless for btree. We could have more regression tests to make the
current behavior clear though. Thoughts from others are welcome.
--
Michael
--
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Bug in comparison of empty jsonb arrays to scalars
2016-11-08 23:31 Bug in comparison of empty jsonb arrays to scalars Nikita Glukhov <[email protected]>
2016-11-09 02:49 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
@ 2016-11-09 18:27 ` Robert Haas <[email protected]>
2016-11-09 22:15 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
0 siblings, 1 reply; 31+ messages in thread
From: Robert Haas @ 2016-11-09 18:27 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Nikita Glukhov <[email protected]>; pgsql-hackers
On Tue, Nov 8, 2016 at 9:49 PM, Michael Paquier
<[email protected]> wrote:
> On Wed, Nov 9, 2016 at 8:31 AM, Nikita Glukhov <[email protected]> wrote:
>> Hi hackers.
>>
>> While working on jsonbstatistics, I found the following bug:
>> an empty jsonb array is considered to be lesser than any scalar,
>> but it is expected that objects > arrays > scalars.
>
> Sources?
How about "our documentation"?
https://www.postgresql.org/docs/current/static/datatype-json.html
Look at the last page.
--
Robert Haas
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
--
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Bug in comparison of empty jsonb arrays to scalars
2016-11-08 23:31 Bug in comparison of empty jsonb arrays to scalars Nikita Glukhov <[email protected]>
2016-11-09 02:49 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
2016-11-09 18:27 ` Re: Bug in comparison of empty jsonb arrays to scalars Robert Haas <[email protected]>
@ 2016-11-09 22:15 ` Michael Paquier <[email protected]>
2016-11-09 22:37 ` Re: Bug in comparison of empty jsonb arrays to scalars Tom Lane <[email protected]>
0 siblings, 1 reply; 31+ messages in thread
From: Michael Paquier @ 2016-11-09 22:15 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Nikita Glukhov <[email protected]>; pgsql-hackers
On Thu, Nov 10, 2016 at 3:27 AM, Robert Haas <[email protected]> wrote:
> On Tue, Nov 8, 2016 at 9:49 PM, Michael Paquier
> <[email protected]> wrote:
>> On Wed, Nov 9, 2016 at 8:31 AM, Nikita Glukhov <[email protected]> wrote:
>>> Hi hackers.
>>>
>>> While working on jsonbstatistics, I found the following bug:
>>> an empty jsonb array is considered to be lesser than any scalar,
>>> but it is expected that objects > arrays > scalars.
>>
>> Sources?
>
> How about "our documentation"?
>
> https://www.postgresql.org/docs/current/static/datatype-json.html
Indeed, I missed that. So that's broken...
--
Michael
--
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Bug in comparison of empty jsonb arrays to scalars
2016-11-08 23:31 Bug in comparison of empty jsonb arrays to scalars Nikita Glukhov <[email protected]>
2016-11-09 02:49 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
2016-11-09 18:27 ` Re: Bug in comparison of empty jsonb arrays to scalars Robert Haas <[email protected]>
2016-11-09 22:15 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
@ 2016-11-09 22:37 ` Tom Lane <[email protected]>
2016-11-10 06:54 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
0 siblings, 1 reply; 31+ messages in thread
From: Tom Lane @ 2016-11-09 22:37 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Robert Haas <[email protected]>; Nikita Glukhov <[email protected]>; pgsql-hackers
Michael Paquier <[email protected]> writes:
> On Thu, Nov 10, 2016 at 3:27 AM, Robert Haas <[email protected]> wrote:
>> https://www.postgresql.org/docs/current/static/datatype-json.html
> Indeed, I missed that. So that's broken...
Given that nobody actually cares what that sort order is, I think that
having to jump through hoops in pg_upgrade in order to fix it is not a
great tradeoff. I suggest changing the documentation to match the code.
regards, tom lane
--
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Bug in comparison of empty jsonb arrays to scalars
2016-11-08 23:31 Bug in comparison of empty jsonb arrays to scalars Nikita Glukhov <[email protected]>
2016-11-09 02:49 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
2016-11-09 18:27 ` Re: Bug in comparison of empty jsonb arrays to scalars Robert Haas <[email protected]>
2016-11-09 22:15 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
2016-11-09 22:37 ` Re: Bug in comparison of empty jsonb arrays to scalars Tom Lane <[email protected]>
@ 2016-11-10 06:54 ` Michael Paquier <[email protected]>
2016-11-10 07:53 ` Re: Bug in comparison of empty jsonb arrays to scalars Ali Akbar <[email protected]>
2016-11-10 09:25 ` Re: Bug in comparison of empty jsonb arrays to scalars Nikita Glukhov <[email protected]>
2016-11-10 14:44 ` Re: Bug in comparison of empty jsonb arrays to scalars Tom Lane <[email protected]>
0 siblings, 3 replies; 31+ messages in thread
From: Michael Paquier @ 2016-11-10 06:54 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Robert Haas <[email protected]>; Nikita Glukhov <[email protected]>; pgsql-hackers
On Thu, Nov 10, 2016 at 7:37 AM, Tom Lane <[email protected]> wrote:
> Michael Paquier <[email protected]> writes:
>> On Thu, Nov 10, 2016 at 3:27 AM, Robert Haas <[email protected]> wrote:
>>> https://www.postgresql.org/docs/current/static/datatype-json.html
>
>> Indeed, I missed that. So that's broken...
>
> Given that nobody actually cares what that sort order is, I think that
> having to jump through hoops in pg_upgrade in order to fix it is not a
> great tradeoff. I suggest changing the documentation to match the code.
Yes, definitely.
=# create table json_data (a jsonb);
CREATE TABLE
=# INSERT INTO json_data values ('{}'::jsonb), ('[]'::jsonb),
('null'::jsonb), ('true'::jsonb), ('1'::jsonb), ('""'::jsonb);
INSERT 0 6
=# SELECT * FROM json_data ORDER BY 1 DESC;
a
------
{}
true
1
""
null
[]
(6 rows)
So that's object > boolean > integer > string > NULL > array.
And attached is a patch.
--
Michael
diff --git a/doc/src/sgml/json.sgml b/doc/src/sgml/json.sgml
index 3cf78d6..b2688ff 100644
--- a/doc/src/sgml/json.sgml
+++ b/doc/src/sgml/json.sgml
@@ -542,7 +542,7 @@ SELECT jdoc->'guid', jdoc->'name' FROM api WHERE jdoc @> '{"tags": ["qu
The <literal>btree</> ordering for <type>jsonb</> datums is seldom
of great interest, but for completeness it is:
<synopsis>
-<replaceable>Object</replaceable> > <replaceable>Array</replaceable> > <replaceable>Boolean</replaceable> > <replaceable>Number</replaceable> > <replaceable>String</replaceable> > <replaceable>Null</replaceable>
+<replaceable>Object</replaceable> > <replaceable>Boolean</replaceable> > <replaceable>Number</replaceable> > <replaceable>String</replaceable> > <replaceable>Null</replaceable> > <replaceable>Array</replaceable>
<replaceable>Object with n pairs</replaceable> > <replaceable>object with n - 1 pairs</replaceable>
--
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Attachments:
[text/plain] json-ordering.patch (917B, ../../CAB7nPqSLmLi+VS7XHHRfs_2e=-viSVW4m=nZm1nMiz-qjOteoA@mail.gmail.com/2-json-ordering.patch)
download | inline diff:
diff --git a/doc/src/sgml/json.sgml b/doc/src/sgml/json.sgml
index 3cf78d6..b2688ff 100644
--- a/doc/src/sgml/json.sgml
+++ b/doc/src/sgml/json.sgml
@@ -542,7 +542,7 @@ SELECT jdoc->'guid', jdoc->'name' FROM api WHERE jdoc @> '{"tags": ["qu
The <literal>btree</> ordering for <type>jsonb</> datums is seldom
of great interest, but for completeness it is:
<synopsis>
-<replaceable>Object</replaceable> > <replaceable>Array</replaceable> > <replaceable>Boolean</replaceable> > <replaceable>Number</replaceable> > <replaceable>String</replaceable> > <replaceable>Null</replaceable>
+<replaceable>Object</replaceable> > <replaceable>Boolean</replaceable> > <replaceable>Number</replaceable> > <replaceable>String</replaceable> > <replaceable>Null</replaceable> > <replaceable>Array</replaceable>
<replaceable>Object with n pairs</replaceable> > <replaceable>object with n - 1 pairs</replaceable>
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Bug in comparison of empty jsonb arrays to scalars
2016-11-08 23:31 Bug in comparison of empty jsonb arrays to scalars Nikita Glukhov <[email protected]>
2016-11-09 02:49 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
2016-11-09 18:27 ` Re: Bug in comparison of empty jsonb arrays to scalars Robert Haas <[email protected]>
2016-11-09 22:15 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
2016-11-09 22:37 ` Re: Bug in comparison of empty jsonb arrays to scalars Tom Lane <[email protected]>
2016-11-10 06:54 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
@ 2016-11-10 07:53 ` Ali Akbar <[email protected]>
2 siblings, 0 replies; 31+ messages in thread
From: Ali Akbar @ 2016-11-10 07:53 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Tom Lane <[email protected]>; Robert Haas <[email protected]>; Nikita Glukhov <[email protected]>; pgsql-hackers
2016-11-10 13:54 GMT+07:00 Michael Paquier <[email protected]>:
> On Thu, Nov 10, 2016 at 7:37 AM, Tom Lane <[email protected]> wrote:
> > Given that nobody actually cares what that sort order is, I think that
> > having to jump through hoops in pg_upgrade in order to fix it is not a
> > great tradeoff. I suggest changing the documentation to match the code.
>
Don't you in this case think we should match sort order in javascript?
> Yes, definitely.
> =# create table json_data (a jsonb);
> CREATE TABLE
> =# INSERT INTO json_data values ('{}'::jsonb), ('[]'::jsonb),
> ('null'::jsonb), ('true'::jsonb), ('1'::jsonb), ('""'::jsonb);
> INSERT 0 6
> =# SELECT * FROM json_data ORDER BY 1 DESC;
> a
> ------
> {}
> true
> 1
> ""
> null
> []
> (6 rows)
> So that's object > boolean > integer > string > NULL > array.
>
> a = [{}, [], null, true, 1, '""']
[ {}, [], null, true, 1, '""' ]
> a.sort()
[ [], '""', 1, {}, null, true ]
> a.reverse()
[ true, null, {}, 1, '""', [] ]
So in this case it's boolean > NULL > Object > integer > string > array
(tried in Chromium 53, Firefox 49 and Node v6.9.1)
When I tried to search for the ECMA Standard for this behavior, i found
this: http://blog.rodneyrehm.de/archives/14-Sorting-Were-Doing-It-Wrong.html.
There are problems about automatic conversion in javascript, like this:
> a = [{}, [], null, true, 1, 'someotherstring']
[ {}, [], null, true, 1, 'someotherstring' ]
> a.sort().reverse()
[ true, 'someotherstring', null, {}, 1, [] ]
versus this:
> a = [{}, [], null, true, 1, 'SomeOtherString']
[ {}, [], null, true, 1, 'SomeOtherString' ]
> a.sort().reverse()
[ true, null, {}, 'SomeOtherString', 1, [] ]
and this:
> a = [{}, [], null, true, 1, '2']
[ {}, [], null, true, 1, '2' ]
> a.sort().reverse()
[ true, null, {}, '2', 1, [] ]
So we can't replicate javascript sort order without emulating those.
Regards,
Ali Akbar
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Bug in comparison of empty jsonb arrays to scalars
2016-11-08 23:31 Bug in comparison of empty jsonb arrays to scalars Nikita Glukhov <[email protected]>
2016-11-09 02:49 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
2016-11-09 18:27 ` Re: Bug in comparison of empty jsonb arrays to scalars Robert Haas <[email protected]>
2016-11-09 22:15 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
2016-11-09 22:37 ` Re: Bug in comparison of empty jsonb arrays to scalars Tom Lane <[email protected]>
2016-11-10 06:54 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
@ 2016-11-10 09:25 ` Nikita Glukhov <[email protected]>
2 siblings, 0 replies; 31+ messages in thread
From: Nikita Glukhov @ 2016-11-10 09:25 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: pgsql-hackers
On 10.11.2016 09:54, Michael Paquier wrote:
> Yes, definitely.
> =# create table json_data (a jsonb);
> CREATE TABLE
> =# INSERT INTO json_data values ('{}'::jsonb), ('[]'::jsonb),
> ('null'::jsonb), ('true'::jsonb), ('1'::jsonb), ('""'::jsonb);
> INSERT 0 6
> =# SELECT * FROM json_data ORDER BY 1 DESC;
> a
> ------
> {}
> true
> 1
> ""
> null
> []
> (6 rows)
> So that's object > boolean > integer > string > NULL > array.
>
> And attached is a patch.
Perhaps I did not explain it clearly enough, but only *empty top-level*
arrays are out of the correct order.
See complete example:
=# SELECT * FROM (VALUES
('null'::jsonb), ('0'), ('""'), ('true'), ('[]'), ('{}'),
('[null]'), ('[0]'), ('[""]'), ('[true]'), ('[[]]'), ('[{}]'),
('{"a": null}'), ('{"a": 0}'), ('{"a": ""}'), ('{"a": true}'),
('{"a": []}'), ('{"a": {}}')
) valsORDER BY 1;
column1
-------------
[]
null
""
0
true
[null]
[""]
[0]
[true]
[[]]
[{}]
{}
{"a": null}
{"a": ""}
{"a": 0}
{"a": true}
{"a": []}
{"a": {}}
(18 rows)
--
Nikita Glukhov
Postgres Professional: http://www.postgrespro.com
The Russian Postgres Company
--
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Bug in comparison of empty jsonb arrays to scalars
2016-11-08 23:31 Bug in comparison of empty jsonb arrays to scalars Nikita Glukhov <[email protected]>
2016-11-09 02:49 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
2016-11-09 18:27 ` Re: Bug in comparison of empty jsonb arrays to scalars Robert Haas <[email protected]>
2016-11-09 22:15 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
2016-11-09 22:37 ` Re: Bug in comparison of empty jsonb arrays to scalars Tom Lane <[email protected]>
2016-11-10 06:54 ` Re: Bug in comparison of empty jsonb arrays to scalars Michael Paquier <[email protected]>
@ 2016-11-10 14:44 ` Tom Lane <[email protected]>
2 siblings, 0 replies; 31+ messages in thread
From: Tom Lane @ 2016-11-10 14:44 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Robert Haas <[email protected]>; Nikita Glukhov <[email protected]>; pgsql-hackers
Michael Paquier <[email protected]> writes:
> On Thu, Nov 10, 2016 at 7:37 AM, Tom Lane <[email protected]> wrote:
>> Given that nobody actually cares what that sort order is, I think that
>> having to jump through hoops in pg_upgrade in order to fix it is not a
>> great tradeoff. I suggest changing the documentation to match the code.
> Yes, definitely.
> So that's object > boolean > integer > string > NULL > array.
No, because the issue is that empty and nonempty arrays sort differently.
regression=# create table json_data (a jsonb);
CREATE TABLE
regression=# INSERT INTO json_data values ('{}'::jsonb), ('[]'::jsonb),
regression-# ('null'::jsonb), ('true'::jsonb), ('1'::jsonb), ('""'::jsonb),
regression-# ('[42]'::jsonb),('[[43]]'::jsonb);
INSERT 0 8
regression=# SELECT * FROM json_data ORDER BY 1 DESC;
a
--------
{}
[[43]]
[42]
true
1
""
null
[]
(8 rows)
> And attached is a patch.
If we go with the fix-the-docs approach, we'll need to have two entries
for empty and nonempty arrays. It's definitely ugly. Still, my judgement
is that it's not worth the pain of changing the behavior. It was never
intended that this sort order be anything but an implementation detail.
(I guess another approach is to not document the order at all ...)
regards, tom lane
--
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* [PATCH 5/8] bloom fixes and tweaks
@ 2021-01-12 23:59 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Tomas Vondra @ 2021-01-12 23:59 UTC (permalink / raw)
---
doc/src/sgml/ref/create_index.sgml | 2 +-
src/backend/access/brin/brin_bloom.c | 84 ++++++++++++++----------
src/test/regress/expected/brin_bloom.out | 14 ++--
src/test/regress/sql/brin_bloom.sql | 6 +-
4 files changed, 60 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 8db10b7b1e..244e5834d8 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -573,7 +573,7 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class=
equal to -1, the number of distinct non-null is assumed linear with
the maximum possible number of tuples in the block range (about 290
rows per block). The default values is <literal>-0.1</literal>, and
- the minimum number of distinct non-null values is <literal>128</literal>.
+ the minimum number of distinct non-null values is <literal>16</literal>.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c
index b7aa6d9f11..ffeb459d3e 100644
--- a/src/backend/access/brin/brin_bloom.c
+++ b/src/backend/access/brin/brin_bloom.c
@@ -8,12 +8,12 @@
*
* A BRIN opclass summarizing page range into a bloom filter.
*
- * Bloom filters allow efficient test whether a given page range contains
+ * Bloom filters allow efficient testing whether a given page range contains
* a particular value. Therefore, if we summarize each page range into a
- * bloom filter, we can easily and cheaply test wheter it containst values
+ * bloom filter, we can easily and cheaply test wheter it contains values
* we get later.
*
- * The index only supports equality operator, similarly to hash indexes.
+ * The index only supports equality operators, similarly to hash indexes.
* BRIN bloom indexes are however much smaller, and support only bitmap
* scans.
*
@@ -51,9 +51,9 @@
* the bloom filter. On the other hand, we want to keep the index as small
* as possible - that's one of the basic advantages of BRIN indexes.
*
- * The number of distinct elements (in a page range) depends on the data,
- * we can consider it fixed. This simplifies the trade-off to just false
- * positive rate vs. size.
+ * Although the number of distinct elements (in a page range) depends on
+ * the data, we can consider it fixed. This simplifies the trade-off to
+ * just false positive rate vs. size.
*
* At the page range level, false positive rate is a probability the bloom
* filter matches a random value. For the whole index (with sufficiently
@@ -65,7 +65,7 @@
* the bitmap is inherently random, compression can't reliably help here.
* To reduce the size of a filter (to fit to a page), we have to either
* accept higher false positive rate (undesirable), or reduce the number
- * of distinct items to be stored in the filter. We can't quite the input
+ * of distinct items to be stored in the filter. We can't alter the input
* data, of course, but we may make the BRIN page ranges smaller - instead
* of the default 128 pages (1MB) we may build index with 16-page ranges,
* or something like that. This does help even for random data sets, as
@@ -87,7 +87,8 @@
* not entirely clear how to distrubute the space between those columns.
*
* The current logic, implemented in brin_bloom_get_ndistinct, attempts to
- * make some basic sizing decisions, based on the table ndistinct estimate.
+ * make some basic sizing decisions, based on the size of BRIN ranges, and
+ * the maximum number of rows per range.
*
*
* sort vs. hash
@@ -200,10 +201,20 @@ typedef struct BloomOptions
/*
* Allowed range and default value for the false positive range. The exact
- * values are somewhat arbitrary.
+ * values are somewhat arbitrary, but were chosen considering the various
+ * parameters (size of filter vs. page size, etc.).
+ *
+ * The lower the false-positive rate, the more accurate the filter is, but
+ * it also gets larger - at some point this eliminates the main advantage
+ * of BRIN indexes, which is the tiny size. At 0.01% the index is about
+ * 10% of the table (assuming 290 distinct values per 8kB page).
+ *
+ * On the other hand, as the false-positive rate increases, larger part of
+ * the table has to be scanned due to mismatches - at 25% we're probably
+ * close to sequential scan being cheaper.
*/
-#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.001 /* 0.1% fp rate */
-#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.1 /* 10% fp rate */
+#define BLOOM_MIN_FALSE_POSITIVE_RATE 0.0001 /* 0.01% fp rate */
+#define BLOOM_MAX_FALSE_POSITIVE_RATE 0.25 /* 25% fp rate */
#define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */
#define BloomGetNDistinctPerRange(opts) \
@@ -302,11 +313,21 @@ bloom_init(int ndistinct, double false_positive_rate)
Assert(ndistinct > 0);
Assert((false_positive_rate > 0) && (false_positive_rate < 1.0));
- m = ceil((ndistinct * log(false_positive_rate)) / log(1.0 / (pow(2.0, log(2.0)))));
+ /* sizing bloom filter: -(n * ln(p)) / (ln(2))^2 */
+ m = ceil(- (ndistinct * log(false_positive_rate)) / pow(log(2.0), 2));
/* round m to whole bytes */
m = ((m + 7) / 8) * 8;
+ /*
+ * Reject filters that are obviously too large to store on a page.
+ *
+ * We do expect the bloom filter to eventually switch to hashing mode,
+ * and it's bound to be almost perfectly random, so not compressible.
+ */
+ if ((m/8) > BLCKSZ)
+ elog(ERROR, "the bloom filter is too large (%d > %d)", (m/8), BLCKSZ);
+
/*
* round(log(2.0) * m / ndistinct), but assume round() may not be
* available on Windows
@@ -315,16 +336,10 @@ bloom_init(int ndistinct, double false_positive_rate)
k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k);
/*
- * Allocate the bloom filter with a minimum size 64B (about 40B in the
- * bitmap part). We require space at least for the header.
- *
- * XXX Maybe the 64B min size is not really needed?
+ * Allocate the bloom filter (initially it's just a header, we'll make
+ * it larger as needed).
*/
- len = Max(offsetof(BloomFilter, data), 64);
-
- /* Reject filters that are obviously too large to store on a page. */
- if (len > BLCKSZ)
- elog(ERROR, "the bloom filter is too large (%zu > %d)", len, BLCKSZ);
+ len = offsetof(BloomFilter, data);
filter = (BloomFilter *) palloc0(len);
@@ -686,7 +701,7 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* brin_bloom_get_ndistinct
* Determine the ndistinct value used to size bloom filter.
*
- * Tweak the ndistinct value based on the pagesPerRange value. First,
+ * Adjust the ndistinct value based on the pagesPerRange value. First,
* if it's negative, it's assumed to be relative to maximum number of
* tuples in the range (assuming each page gets MaxHeapTuplesPerPage
* tuples, which is likely a significant over-estimate). We also clamp
@@ -701,6 +716,10 @@ brin_bloom_opcinfo(PG_FUNCTION_ARGS)
* and compute the expected number of distinct values in a range. But
* that may be tricky due to data being sorted in various ways, so it
* seems better to rely on the upper estimate.
+ *
+ * XXX We might also calculate a better estimate of rows per BRIN range,
+ * instead of using MaxHeapTuplesPerPage (which probably produces values
+ * much higher than reality).
*/
static int
brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
@@ -738,11 +757,6 @@ brin_bloom_get_ndistinct(BrinDesc *bdesc, BloomOptions *opts)
return (int) ndistinct;
}
-static double
-brin_bloom_get_fp_rate(BrinDesc *bdesc, BloomOptions *opts)
-{
- return BloomGetFalsePositiveRate(opts);
-}
/*
* Examine the given index tuple (which contains partial status of a certain
@@ -777,7 +791,7 @@ brin_bloom_add_value(PG_FUNCTION_ARGS)
if (column->bv_allnulls)
{
filter = bloom_init(brin_bloom_get_ndistinct(bdesc, opts),
- brin_bloom_get_fp_rate(bdesc, opts));
+ BloomGetFalsePositiveRate(opts));
column->bv_values[0] = PointerGetDatum(filter);
column->bv_allnulls = false;
updated = true;
@@ -949,7 +963,7 @@ brin_bloom_union(PG_FUNCTION_ARGS)
* Cache and return inclusion opclass support procedure
*
* Return the procedure corresponding to the given function support number
- * or null if it is not exists.
+ * or null if it does not exists.
*/
static FmgrInfo *
bloom_get_procinfo(BrinDesc *bdesc, uint16 attno, uint16 procnum)
@@ -1050,11 +1064,8 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
initStringInfo(&str);
appendStringInfoChar(&str, '{');
- /*
- * XXX not sure the detoasting is necessary (probably not, this
- * can only be in an index).
- */
- filter = (BloomFilter *) PG_DETOAST_DATUM(PG_GETARG_BYTEA_PP(0));
+ /* Detoasting not needed (this can only be in an index). */
+ filter = (BloomFilter *) PG_GETARG_BYTEA_PP(0);
if (BLOOM_IS_HASHED(filter))
{
@@ -1063,9 +1074,12 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS)
}
else
{
+ /*
+ * XXX Maybe include the sorted/unsorted values? Seems a bit too
+ * much useless detail (internal hash values).
+ */
appendStringInfo(&str, "mode: sorted nvalues: %u nsorted: %u",
filter->nvalues, filter->nsorted);
- /* TODO include the sorted/unsorted values */
}
appendStringInfoChar(&str, '}');
diff --git a/src/test/regress/expected/brin_bloom.out b/src/test/regress/expected/brin_bloom.out
index 19b866283a..24ea5f6e42 100644
--- a/src/test/regress/expected/brin_bloom.out
+++ b/src/test/regress/expected/brin_bloom.out
@@ -58,17 +58,17 @@ CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
);
ERROR: value -1.1 out of bounds for option "n_distinct_per_range"
DETAIL: Valid values are between "-1.000000" and "2147483647.000000".
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
-ERROR: value 0.0009 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.00009 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
-ERROR: value 0.11 out of bounds for option "false_positive_rate"
-DETAIL: Valid values are between "0.001000" and "0.100000".
+ERROR: value 0.26 out of bounds for option "false_positive_rate"
+DETAIL: Valid values are between "0.000100" and "0.250000".
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops,
charcol char_bloom_ops,
diff --git a/src/test/regress/sql/brin_bloom.sql b/src/test/regress/sql/brin_bloom.sql
index 3c2ef56316..d587f3962f 100644
--- a/src/test/regress/sql/brin_bloom.sql
+++ b/src/test/regress/sql/brin_bloom.sql
@@ -59,12 +59,12 @@ FROM tenk1 ORDER BY thousand, tenthous LIMIT 25;
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
byteacol bytea_bloom_ops(n_distinct_per_range = -1.1)
);
--- false_positive_rate must be between 0.001 and 1.0
+-- false_positive_rate must be between 0.0001 and 0.25
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.0009)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.00009)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
- byteacol bytea_bloom_ops(false_positive_rate = 0.11)
+ byteacol bytea_bloom_ops(false_positive_rate = 0.26)
);
CREATE INDEX brinidx_bloom ON brintest_bloom USING brin (
--
2.26.2
--------------CF71AF65F7C337C37C24B045
Content-Type: text/x-patch; charset=UTF-8;
name="0006-add-sort_mode-opclass-parameter-20210112.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0006-add-sort_mode-opclass-parameter-20210112.patch"
^ permalink raw reply [nested|flat] 31+ messages in thread
* Re: Allow parallel plan for referential integrity checks?
@ 2022-02-14 10:56 Frédéric Yhuel <[email protected]>
0 siblings, 0 replies; 31+ messages in thread
From: Frédéric Yhuel @ 2022-02-14 10:56 UTC (permalink / raw)
To: [email protected]
On 2/11/22 00:16, Andreas Karlsson wrote:
> On 2/7/22 11:26, Frédéric Yhuel wrote:
>> Attached is a (naive) patch that aims to fix the case of a FK
>> addition, but the handling of the flag CURSOR_OPT_PARALLEL_OK,
>> generally speaking, looks rather hackish.
>
> Thanks, for the patch. You can add it to the current open commitfest
> (https://commitfest.postgresql.org/37/).
>
OK, I just did. Please let me know if I did something wrong.
Best regards,
Frédéric
^ permalink raw reply [nested|flat] 31+ messages in thread
end of thread, other threads:[~2022-02-14 10:56 UTC | newest]
Thread overview: 31+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2016-11-08 23:31 Bug in comparison of empty jsonb arrays to scalars Nikita Glukhov <[email protected]>
2016-11-09 02:49 ` Michael Paquier <[email protected]>
2016-11-09 18:27 ` Robert Haas <[email protected]>
2016-11-09 22:15 ` Michael Paquier <[email protected]>
2016-11-09 22:37 ` Tom Lane <[email protected]>
2016-11-10 06:54 ` Michael Paquier <[email protected]>
2016-11-10 07:53 ` Ali Akbar <[email protected]>
2016-11-10 09:25 ` Nikita Glukhov <[email protected]>
2016-11-10 14:44 ` Tom Lane <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2021-01-12 23:59 [PATCH 5/8] bloom fixes and tweaks Tomas Vondra <[email protected]>
2022-02-14 10:56 Re: Allow parallel plan for referential integrity checks? Frédéric Yhuel <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox