agora inbox for [email protected]
help / color / mirror / Atom feedFrom: Heikki Linnakangas <[email protected]>
Subject: [PATCH 2/2] Andrey Borodin's test_blockset tool, adapted for SparseBitset
Date: Wed, 13 Mar 2019 21:08:35 +0200
WIP: I don't know if we want to commit something like this in the end, but
it's pretty useful for manual testing.
from https://www.postgresql.org/message-id/FD68CBB1-716F-4344-9E3F-C27C14B8678B%40yandex-team.ru
---
src/test/modules/test_blockset/.gitignore | 4 +
src/test/modules/test_blockset/Makefile | 21 ++
src/test/modules/test_blockset/README | 8 +
.../test_blockset/expected/test_blockset.out | 7 +
.../test_blockset/sql/test_blockset.sql | 3 +
.../test_blockset/test_blockset--1.0.sql | 8 +
.../modules/test_blockset/test_blockset.c | 236 ++++++++++++++++++
.../test_blockset/test_blockset.control | 4 +
src/test/regress/expected/gist.out | 4 +-
src/test/regress/sql/gist.sql | 4 +-
10 files changed, 293 insertions(+), 6 deletions(-)
create mode 100644 src/test/modules/test_blockset/.gitignore
create mode 100644 src/test/modules/test_blockset/Makefile
create mode 100644 src/test/modules/test_blockset/README
create mode 100644 src/test/modules/test_blockset/expected/test_blockset.out
create mode 100644 src/test/modules/test_blockset/sql/test_blockset.sql
create mode 100644 src/test/modules/test_blockset/test_blockset--1.0.sql
create mode 100644 src/test/modules/test_blockset/test_blockset.c
create mode 100644 src/test/modules/test_blockset/test_blockset.control
diff --git a/src/test/modules/test_blockset/.gitignore b/src/test/modules/test_blockset/.gitignore
new file mode 100644
index 00000000000..5dcb3ff9723
--- /dev/null
+++ b/src/test/modules/test_blockset/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/test_blockset/Makefile b/src/test/modules/test_blockset/Makefile
new file mode 100644
index 00000000000..091cf8c0958
--- /dev/null
+++ b/src/test/modules/test_blockset/Makefile
@@ -0,0 +1,21 @@
+# src/test/modules/test_blockset/Makefile
+
+MODULE_big = test_blockset
+OBJS = test_blockset.o $(WIN32RES)
+PGFILEDESC = "test_blockset - test code for block set library"
+
+EXTENSION = test_blockset
+DATA = test_blockset--1.0.sql
+
+REGRESS = test_blockset
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_blockset
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_blockset/README b/src/test/modules/test_blockset/README
new file mode 100644
index 00000000000..730951ff03c
--- /dev/null
+++ b/src/test/modules/test_blockset/README
@@ -0,0 +1,8 @@
+test_blockset overview
+=========================
+
+test_blockset is a test harness module for testing block set data structure.
+There are two main tests:
+1. Test of comliance with Bitmapset on numbers avaiable to Bitmapset
+2. Test of numbers that can exceed INT32_MAX but are just shifted on one bit
+from numbers kept in Bitmapset
\ No newline at end of file
diff --git a/src/test/modules/test_blockset/expected/test_blockset.out b/src/test/modules/test_blockset/expected/test_blockset.out
new file mode 100644
index 00000000000..02c29d5fc0c
--- /dev/null
+++ b/src/test/modules/test_blockset/expected/test_blockset.out
@@ -0,0 +1,7 @@
+CREATE EXTENSION test_blockset;
+SELECT test_blockset();
+ test_blockset
+---------------
+
+(1 row)
+
diff --git a/src/test/modules/test_blockset/sql/test_blockset.sql b/src/test/modules/test_blockset/sql/test_blockset.sql
new file mode 100644
index 00000000000..85d2886676e
--- /dev/null
+++ b/src/test/modules/test_blockset/sql/test_blockset.sql
@@ -0,0 +1,3 @@
+CREATE EXTENSION test_blockset;
+
+SELECT test_blockset();
\ No newline at end of file
diff --git a/src/test/modules/test_blockset/test_blockset--1.0.sql b/src/test/modules/test_blockset/test_blockset--1.0.sql
new file mode 100644
index 00000000000..04eea8a6146
--- /dev/null
+++ b/src/test/modules/test_blockset/test_blockset--1.0.sql
@@ -0,0 +1,8 @@
+/* src/test/modules/test_blockset/test_blockset--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION test_blockset" to load this file. \quit
+
+CREATE FUNCTION test_blockset()
+RETURNS pg_catalog.void STRICT
+AS 'MODULE_PATHNAME' LANGUAGE C;
diff --git a/src/test/modules/test_blockset/test_blockset.c b/src/test/modules/test_blockset/test_blockset.c
new file mode 100644
index 00000000000..218942a587b
--- /dev/null
+++ b/src/test/modules/test_blockset/test_blockset.c
@@ -0,0 +1,236 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_blockset.c
+ * Test block set data structure.
+ *
+ * Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/test/modules/test_blockset/test_blockset.c
+ *
+ * -------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "fmgr.h"
+#include "lib/sparsebitset.h"
+#include "nodes/bitmapset.h"
+#include "utils/memutils.h"
+#include "storage/block.h"
+#include "miscadmin.h"
+
+PG_MODULE_MAGIC;
+
+PG_FUNCTION_INFO_V1(test_blockset);
+
+static void test_blockset_bms_compliance(int32_t);
+static void test_blockset_big_block_numbers(int32_t);
+
+/*
+ * SQL-callable entry point to perform all tests.
+ */
+Datum
+test_blockset(PG_FUNCTION_ARGS)
+{
+ test_blockset_bms_compliance(0);
+ test_blockset_bms_compliance(1);
+ test_blockset_bms_compliance(2);
+ test_blockset_bms_compliance(1337);
+ test_blockset_bms_compliance(100000);
+ test_blockset_big_block_numbers(1337);
+ test_blockset_big_block_numbers(31337);
+ test_blockset_big_block_numbers(5000000);
+ PG_RETURN_VOID();
+}
+
+/*
+ * This test creates random bitmap with test_limit members
+ * and checks that block set behavior is similar to Bitmapset
+ */
+static void test_blockset_bms_compliance(int32_t test_limit)
+{
+ SparseBitset *sbs = NULL;
+ Bitmapset *bms = NULL;
+ BlockNumber blockno;
+ int index;
+ int32_t point_index = 0;
+
+ sbs = sbs_create();
+
+ for (int32_t i = 0; i < test_limit; i++)
+ {
+ blockno = random() & INT32_MAX;
+ /* bms does not support block numbers above INT32_MAX */
+ bms = bms_add_member(bms, (int)blockno);
+ }
+
+ /*
+ * SparseBitset requires that the items are added in-order, so we populate
+ * the SparseBitset after the Bitmapset, by iterating through all the values
+ * from the Bitmapset in the right order.
+ */
+ {
+ int x = -1;
+
+ while ((x = bms_next_member(bms, x)) >= 0)
+ sbs_add_member(sbs, (BlockNumber) (x));
+ }
+
+ index = -1;
+ blockno = InvalidBlockNumber;
+
+ sbs_begin_iterate(sbs);
+ while (true)
+ {
+ bool found;
+ BlockNumber next_bn = sbs_iterate_next(sbs, &found);
+ int next_index = bms_next_member(bms, index);
+
+ if (!found)
+ next_bn = InvalidBlockNumber;
+
+ point_index++;
+
+ if (next_bn == InvalidBlockNumber && next_index == -2)
+ return; /* We have found everything */
+
+ if (((BlockNumber)next_index) != next_bn)
+ {
+ elog(ERROR,
+ "Bitmapset returned value %X different from block set %X,"
+ " test_limit %d, point index %d",
+ next_index, next_bn, test_limit, point_index);
+ }
+
+ if (!sbs_is_member(sbs, next_bn))
+ {
+ elog(ERROR,
+ "Block set did not found present item %X"
+ " test_limit %d, point index %d",
+ next_bn, test_limit, point_index);
+ }
+
+ index = next_index;
+ blockno = next_bn;
+ }
+
+ for (int32_t i = 0; i < test_limit; i++)
+ {
+ blockno = random() & INT32_MAX;
+ if (sbs_is_member(sbs, blockno) != bms_is_member((int)blockno, bms))
+ {
+ elog(ERROR,
+ "Block set did agree with bitmapset item %X"
+ " test_limit %d, point index %d",
+ blockno, test_limit, point_index);
+ }
+ }
+
+ sbs_free(sbs);
+ bms_free(bms);
+}
+
+/*
+ * This test is similar to test_blockset_bms_compliance()
+ * except that it shifts BlockNumbers by one bit to ensure that blockset
+ * operates correctly on values higher that INT32_MAX
+ * This function is copy-pasted from previous with the exception of barrel
+ * shifts for BlockNumbers. I've tried various refactorings, but they all
+ * looked ugly.
+ */
+static void test_blockset_big_block_numbers(int32_t test_limit)
+{
+ SparseBitset *sbs;
+ Bitmapset *bms = NULL;
+ BlockNumber blockno;
+ int index;
+ int32_t point_index = 0;
+ MemoryContext old_cxt;
+
+ MemoryContext test_cxt;
+
+ fprintf(stderr, "TEST %d:\n", test_limit);
+
+ test_cxt = AllocSetContextCreate(CurrentMemoryContext,
+ "blockset test",
+ ALLOCSET_SMALL_SIZES);
+ old_cxt = MemoryContextSwitchTo(test_cxt);
+ sbs = sbs_create();
+ MemoryContextSwitchTo(old_cxt);
+
+ for (int32_t i = 0; i < test_limit; i++)
+ {
+ blockno = random() & INT32_MAX;
+ /* bms does not support block numbers above INT32_MAX */
+ bms = bms_add_member(bms, (int)blockno);
+
+ CHECK_FOR_INTERRUPTS();
+ }
+
+ /*
+ * SparseBitset requires that the items are added in-order, so we populate
+ * the SparseBitset after the Bitmapset, by iterating through all the values
+ * from the Bitmapset in the right order.
+ */
+ {
+ int x = -1;
+
+ while ((x = bms_next_member(bms, x)) >= 0)
+ sbs_add_member(sbs, ((BlockNumber) (x)) << 1);
+ }
+
+ MemoryContextStats(test_cxt);
+
+ index = -1;
+ blockno = InvalidBlockNumber;
+
+ sbs_begin_iterate(sbs);
+ while (true)
+ {
+ bool found;
+ BlockNumber next_bn = sbs_iterate_next(sbs, &found);
+ int next_index = bms_next_member(bms, index);
+
+ if (!found)
+ next_bn = InvalidBlockNumber;
+
+ point_index++;
+
+ if (next_bn == InvalidBlockNumber && next_index == -2)
+ break; /* We have found everything */
+
+ if (((BlockNumber)next_index) != (next_bn >> 1))
+ {
+ elog(ERROR,
+ "Bitmapset returned value %X different from block set %X,"
+ " test_limit %d, point index %d",
+ next_index, next_bn, test_limit, point_index);
+ }
+
+ if (!sbs_is_member(sbs, next_bn))
+ {
+ elog(ERROR,
+ "Block set did not found present item %X"
+ " test_limit %d, point index %d",
+ next_bn, test_limit, point_index);
+ }
+
+ index = next_index;
+ blockno = next_bn;
+ }
+
+ for (int32_t i = 0; i < test_limit; i++)
+ {
+ blockno = random() & INT32_MAX;
+ if (sbs_is_member(sbs, blockno << 1) != bms_is_member((int)blockno, bms))
+ {
+ elog(ERROR,
+ "Block set did agree with bitmapset item %X"
+ " test_limit %d, point index %d",
+ blockno, test_limit, point_index);
+ }
+ }
+
+ sbs_free(sbs);
+ bms_free(bms);
+}
diff --git a/src/test/modules/test_blockset/test_blockset.control b/src/test/modules/test_blockset/test_blockset.control
new file mode 100644
index 00000000000..fdb7598c5a7
--- /dev/null
+++ b/src/test/modules/test_blockset/test_blockset.control
@@ -0,0 +1,4 @@
+comment = 'Test code for block set library'
+default_version = '1.0'
+module_pathname = '$libdir/test_blockset'
+relocatable = true
diff --git a/src/test/regress/expected/gist.out b/src/test/regress/expected/gist.out
index f5a2993aaf2..5b92f08c747 100644
--- a/src/test/regress/expected/gist.out
+++ b/src/test/regress/expected/gist.out
@@ -27,9 +27,7 @@ insert into gist_point_tbl (id, p)
select g+100000, point(g*10+1, g*10+1) from generate_series(1, 10000) g;
-- To test vacuum, delete some entries from all over the index.
delete from gist_point_tbl where id % 2 = 1;
--- And also delete some concentration of values. (GiST doesn't currently
--- attempt to delete pages even when they become empty, but if it did, this
--- would exercise it)
+-- And also delete some concentration of values.
delete from gist_point_tbl where id < 10000;
vacuum analyze gist_point_tbl;
-- rebuild the index with a different fillfactor
diff --git a/src/test/regress/sql/gist.sql b/src/test/regress/sql/gist.sql
index bae722fe13c..e66396e851b 100644
--- a/src/test/regress/sql/gist.sql
+++ b/src/test/regress/sql/gist.sql
@@ -28,9 +28,7 @@ select g+100000, point(g*10+1, g*10+1) from generate_series(1, 10000) g;
-- To test vacuum, delete some entries from all over the index.
delete from gist_point_tbl where id % 2 = 1;
--- And also delete some concentration of values. (GiST doesn't currently
--- attempt to delete pages even when they become empty, but if it did, this
--- would exercise it)
+-- And also delete some concentration of values.
delete from gist_point_tbl where id < 10000;
vacuum analyze gist_point_tbl;
--
2.20.1
--------------0CF1D8B78F83F73696708403--
view thread (4+ messages) latest in thread
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected]
Subject: Re: [PATCH 2/2] Andrey Borodin's test_blockset tool, adapted for SparseBitset
In-Reply-To: <no-message-id-1883640@localhost>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox