agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH 1/1] resownerbench 10+ messages / 2 participants [nested] [flat]
* [PATCH 1/1] resownerbench @ 2020-11-11 22:40 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 10+ messages in thread From: Heikki Linnakangas @ 2020-11-11 22:40 UTC (permalink / raw) --- contrib/resownerbench/Makefile | 17 ++ contrib/resownerbench/resownerbench--1.0.sql | 14 ++ contrib/resownerbench/resownerbench.c | 154 +++++++++++++++++++ contrib/resownerbench/resownerbench.control | 6 + contrib/resownerbench/snaptest.sql | 37 +++++ 5 files changed, 228 insertions(+) create mode 100644 contrib/resownerbench/Makefile create mode 100644 contrib/resownerbench/resownerbench--1.0.sql create mode 100644 contrib/resownerbench/resownerbench.c create mode 100644 contrib/resownerbench/resownerbench.control create mode 100644 contrib/resownerbench/snaptest.sql diff --git a/contrib/resownerbench/Makefile b/contrib/resownerbench/Makefile new file mode 100644 index 00000000000..9b0e1cfee1a --- /dev/null +++ b/contrib/resownerbench/Makefile @@ -0,0 +1,17 @@ +MODULE_big = resownerbench +OBJS = resownerbench.o + +EXTENSION = resownerbench +DATA = resownerbench--1.0.sql +PGFILEDESC = "resownerbench - benchmark for ResourceOwners" + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/resownerbench +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/resownerbench/resownerbench--1.0.sql b/contrib/resownerbench/resownerbench--1.0.sql new file mode 100644 index 00000000000..d29182f5982 --- /dev/null +++ b/contrib/resownerbench/resownerbench--1.0.sql @@ -0,0 +1,14 @@ +/* contrib/resownerbench/resownerbench--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION resownerbench" to load this file. \quit + +CREATE FUNCTION snapshotbench_lifo(numkeep int, numsnaps int, numiters int) +RETURNS double precision +AS 'MODULE_PATHNAME', 'snapshotbench_lifo' +LANGUAGE C STRICT VOLATILE; + +CREATE FUNCTION snapshotbench_fifo(numkeep int, numsnaps int, numiters int) +RETURNS double precision +AS 'MODULE_PATHNAME', 'snapshotbench_fifo' +LANGUAGE C STRICT VOLATILE; diff --git a/contrib/resownerbench/resownerbench.c b/contrib/resownerbench/resownerbench.c new file mode 100644 index 00000000000..acfb6c39199 --- /dev/null +++ b/contrib/resownerbench/resownerbench.c @@ -0,0 +1,154 @@ +#include "postgres.h" + +#include "catalog/pg_type.h" +#include "catalog/pg_statistic.h" +#include "executor/spi.h" +#include "funcapi.h" +#include "libpq/pqsignal.h" +#include "utils/catcache.h" +#include "utils/syscache.h" +#include "utils/timestamp.h" +#include "utils/snapmgr.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(snapshotbench_lifo); +PG_FUNCTION_INFO_V1(snapshotbench_fifo); + +/* + * ResourceOwner Performance test, using RegisterSnapshot(). + * + * This takes three parameters: numkeep, numsnaps, numiters. + * + * First, we register 'numkeep' snapshots. They are kept registed + * until the end of the test. Then, we repeatedly register and + * unregister 'numsnaps - numkeep' additional snapshots, repeating + * 'numiters' times. All the register/unregister calls are made in + * LIFO order. + * + * Returns the time spent, in milliseconds. + * + * The idea is to test the performance of ResourceOwnerRemember() + * and ReourceOwnerForget() operations, under different regimes. + * + * In the old implementation, if 'numsnaps' is small enough, all + * the entries fit in the resource owner's small array (it can + * hold 64 entries). + * + * In the new implementation, the array is much smaller, only 8 + * entries, but it's used together with the hash table so that + * we stay in the "array regime" as long as 'numsnaps - numkeep' + * is smaller than 8 entries. + * + * 'numiters' can be adjusted to adjust the overall runtime to be + * suitable long. + */ +Datum +snapshotbench_lifo(PG_FUNCTION_ARGS) +{ + int numkeep = PG_GETARG_INT32(0); + int numsnaps = PG_GETARG_INT32(1); + int numiters = PG_GETARG_INT32(2); + int i; + instr_time start, + duration; + Snapshot lsnap; + Snapshot *rs; + int numregistered = 0; + + rs = palloc(Max(numsnaps, numkeep) * sizeof(Snapshot)); + + lsnap = GetLatestSnapshot(); + + PG_SETMASK(&BlockSig); + INSTR_TIME_SET_CURRENT(start); + + while (numregistered < numkeep) + { + rs[numregistered] = RegisterSnapshot(lsnap); + numregistered++; + } + + for (i = 0 ; i < numiters; i++) + { + while (numregistered < numsnaps) + { + rs[numregistered] = RegisterSnapshot(lsnap); + numregistered++; + } + + while (numregistered > numkeep) + { + numregistered--; + UnregisterSnapshot(rs[numregistered]); + } + } + + while (numregistered > 0) + { + numregistered--; + UnregisterSnapshot(rs[numregistered]); + } + + INSTR_TIME_SET_CURRENT(duration); + INSTR_TIME_SUBTRACT(duration, start); + PG_SETMASK(&UnBlockSig); + + PG_RETURN_FLOAT8(INSTR_TIME_GET_MILLISEC(duration)); +}; + + +/* + * Same, but do the register/unregister operations in + * FIFO order. + */ +Datum +snapshotbench_fifo(PG_FUNCTION_ARGS) +{ + int numkeep = PG_GETARG_INT32(0); + int numsnaps = PG_GETARG_INT32(1); + int numiters = PG_GETARG_INT32(2); + int i, + j; + instr_time start, + duration; + Snapshot lsnap; + Snapshot *rs; + int numregistered = 0; + + rs = palloc(Max(numsnaps, numkeep) * sizeof(Snapshot)); + + lsnap = GetLatestSnapshot(); + + PG_SETMASK(&BlockSig); + INSTR_TIME_SET_CURRENT(start); + + while (numregistered < numkeep) + { + rs[numregistered] = RegisterSnapshot(lsnap); + numregistered++; + } + + for (i = 0 ; i < numiters; i++) + { + while (numregistered < numsnaps) + { + rs[numregistered] = RegisterSnapshot(lsnap); + numregistered++; + } + + for (j = numkeep; j < numregistered; j++) + UnregisterSnapshot(rs[j]); + numregistered = numkeep; + } + + for (j = 0; j < numregistered; j++) + UnregisterSnapshot(rs[j]); + numregistered = numkeep; + + INSTR_TIME_SET_CURRENT(duration); + INSTR_TIME_SUBTRACT(duration, start); + PG_SETMASK(&UnBlockSig); + + PG_RETURN_FLOAT8(INSTR_TIME_GET_MILLISEC(duration)); +}; diff --git a/contrib/resownerbench/resownerbench.control b/contrib/resownerbench/resownerbench.control new file mode 100644 index 00000000000..ada88b8eed8 --- /dev/null +++ b/contrib/resownerbench/resownerbench.control @@ -0,0 +1,6 @@ +# resownerbench + +comment = 'benchmark for ResourceOwners' +default_version = '1.0' +module_pathname = '$libdir/resownerbench' +relocatable = true diff --git a/contrib/resownerbench/snaptest.sql b/contrib/resownerbench/snaptest.sql new file mode 100644 index 00000000000..18c54e13fc9 --- /dev/null +++ b/contrib/resownerbench/snaptest.sql @@ -0,0 +1,37 @@ +-- +-- Performance test RegisterSnapshot/UnregisterSnapshot. +-- +select numkeep, numsnaps, + -- numiters, + -- round(lifo_time_ms) as lifo_total_time_ms, + -- round(fifo_time_ms) as fifo_total_time_ms, + round((lifo_time_ms::numeric / (numkeep + (numsnaps - numkeep) * numiters)) * 1000000, 1) as lifo_time_ns, + round((fifo_time_ms::numeric / (numkeep + (numsnaps - numkeep) * numiters)) * 1000000, 1) as fifo_time_ns +from +(values (0, 1, 10000000), + (0, 5, 2000000), + (0, 10, 1000000), + (0, 60, 100000), + (0, 70, 100000), + (0, 100, 100000), + (0, 1000, 10000), + (0, 10000, 1000), + +-- These tests keep 9 snapshots registered across the iterations. That +-- exceeds the size of the little array in the patch, so this exercises +-- the hash lookups. Without the patch, these still fit in the array +-- (it's 64 entries without the patch) + (9, 10, 10000000), + (9, 100, 100000), + (9, 1000, 10000), + (9, 10000, 1000), + +-- These exceed the 64 entry array even without the patch, so these fall +-- in the hash table regime with and without the patch. + (65, 70, 1000000), + (65, 100, 100000), + (65, 1000, 10000), + (65, 10000, 1000) +) AS params (numkeep, numsnaps, numiters), +lateral snapshotbench_lifo(numkeep, numsnaps, numiters) as lifo_time_ms, +lateral snapshotbench_fifo(numkeep, numsnaps, numiters) as fifo_time_ms; -- 2.29.2 --------------1104342304EC6341F5D7C60C-- ^ permalink raw reply [nested|flat] 10+ messages in thread
* [PATCH v12 14/17] Push BitmapHeapScan prefetch code into heapam.c @ 2024-03-22 13:42 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 10+ messages in thread From: Melanie Plageman @ 2024-03-22 13:42 UTC (permalink / raw) In preparation for transitioning to using the streaming read API for prefetching [1], move all of the BitmapHeapScanState members related to prefetching and the functions for accessing them into the HeapScanDescData and TableScanDescData. Members that still need to be accessed in BitmapHeapNext() could not be moved into heap AM-specific code. Specifically, parallel iterator setup requires several components which seem odd to pass to the table AM API. [1] https://www.postgresql.org/message-id/flat/CA%2BhUKGJkOiOCa%2Bmag4BF%2BzHo7qo%3Do9CFheB8%3Dg6uT5TUm2... --- src/backend/access/heap/heapam.c | 26 ++ src/backend/access/heap/heapam_handler.c | 262 +++++++++++++++++ src/backend/executor/nodeBitmapHeapscan.c | 341 ++-------------------- src/include/access/heapam.h | 17 ++ src/include/access/relscan.h | 8 + src/include/access/tableam.h | 26 +- src/include/nodes/execnodes.h | 14 - 7 files changed, 355 insertions(+), 339 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 27e9d05f14b..086dae59668 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -948,8 +948,16 @@ heap_beginscan(Relation relation, Snapshot snapshot, scan->rs_base.rs_flags = flags; scan->rs_base.rs_parallel = parallel_scan; scan->rs_strategy = NULL; /* set in initscan */ + + scan->rs_base.blockno = InvalidBlockNumber; + scan->rs_vmbuffer = InvalidBuffer; scan->rs_empty_tuples_pending = 0; + scan->pvmbuffer = InvalidBuffer; + + scan->pfblockno = InvalidBlockNumber; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; /* * Disable page-at-a-time mode if it's not a MVCC-safe snapshot. @@ -1032,6 +1040,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, scan->rs_base.rs_flags &= ~SO_ALLOW_PAGEMODE; } + scan->rs_base.blockno = InvalidBlockNumber; + + scan->pfblockno = InvalidBlockNumber; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; + /* * unpin scan buffers */ @@ -1044,6 +1058,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, scan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(scan->pvmbuffer)) + { + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + } + /* * reinitialize scan descriptor */ @@ -1069,6 +1089,12 @@ heap_endscan(TableScanDesc sscan) scan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(scan->pvmbuffer)) + { + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + } + /* * decrement relation reference count and free scan descriptor storage */ diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index bd630e38fa8..36c6fb021c3 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -61,6 +61,9 @@ static bool SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, OffsetNumber tupoffset); static BlockNumber heapam_scan_get_blocks_done(HeapScanDesc hscan); +static inline void BitmapAdjustPrefetchIterator(HeapScanDesc scan); +static inline void BitmapAdjustPrefetchTarget(HeapScanDesc scan); +static inline void BitmapPrefetch(HeapScanDesc scan); static const TableAmRoutine heapam_methods; @@ -2197,6 +2200,73 @@ heapam_estimate_rel_size(Relation rel, int32 *attr_widths, * ------------------------------------------------------------------------ */ +/* + * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator + * + * We keep track of how far the prefetch iterator is ahead of the main + * iterator in prefetch_pages. For each block the main iterator returns, we + * decrement prefetch_pages. + */ +static inline void +BitmapAdjustPrefetchIterator(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + BitmapHeapIterator *prefetch_iterator = scan->rs_base.rs_pf_bhs_iterator; + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + TBMIterateResult tbmpre; + + if (pstate == NULL) + { + if (scan->prefetch_pages > 0) + { + /* The main iterator has closed the distance by one page */ + scan->prefetch_pages--; + } + else if (prefetch_iterator) + { + /* Do not let the prefetch iterator get behind the main one */ + bhs_iterate(prefetch_iterator, &tbmpre); + scan->pfblockno = tbmpre.blockno; + } + return; + } + + /* + * Adjusting the prefetch iterator before invoking + * table_scan_bitmap_next_block() keeps prefetch distance higher across + * the parallel workers. + */ + if (scan->rs_base.prefetch_maximum > 0) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages > 0) + { + pstate->prefetch_pages--; + SpinLockRelease(&pstate->mutex); + } + else + { + /* Release the mutex before iterating */ + SpinLockRelease(&pstate->mutex); + + /* + * In case of shared mode, we can not ensure that the current + * blockno of the main iterator and that of the prefetch iterator + * are same. It's possible that whatever blockno we are + * prefetching will be processed by another process. Therefore, + * we don't validate the blockno here as we do in non-parallel + * case. + */ + if (prefetch_iterator) + { + bhs_iterate(prefetch_iterator, &tbmpre); + scan->pfblockno = tbmpre.blockno; + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_block(TableScanDesc scan, bool *recheck, BlockNumber *blockno, @@ -2215,6 +2285,8 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, *blockno = InvalidBlockNumber; *recheck = true; + BitmapAdjustPrefetchIterator(hscan); + do { CHECK_FOR_INTERRUPTS(); @@ -2355,6 +2427,18 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, else (*exact_pages)++; + /* + * If serial, we can error out if the the prefetch block doesn't stay + * ahead of the current block. + */ + if (scan->bm_parallel == NULL && + scan->rs_pf_bhs_iterator && + hscan->pfblockno < hscan->rs_base.blockno) + elog(ERROR, "prefetch and main iterators are out of sync"); + + /* Adjust the prefetch target */ + BitmapAdjustPrefetchTarget(hscan); + /* * Return true to indicate that a valid block was found and the bitmap is * not exhausted. If there are no visible tuples on this page, @@ -2365,6 +2449,154 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, return true; } +/* + * BitmapAdjustPrefetchTarget - Adjust the prefetch target + * + * Increase prefetch target if it's not yet at the max. Note that + * we will increase it to zero after fetching the very first + * page/tuple, then to one after the second tuple is fetched, then + * it doubles as later pages are fetched. + */ +static inline void +BitmapAdjustPrefetchTarget(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + int prefetch_maximum = scan->rs_base.prefetch_maximum; + + if (pstate == NULL) + { + if (scan->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (scan->prefetch_target >= prefetch_maximum / 2) + scan->prefetch_target = prefetch_maximum; + else if (scan->prefetch_target > 0) + scan->prefetch_target *= 2; + else + scan->prefetch_target++; + return; + } + + /* Do an unlocked check first to save spinlock acquisitions. */ + if (pstate->prefetch_target < prefetch_maximum) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (pstate->prefetch_target >= prefetch_maximum / 2) + pstate->prefetch_target = prefetch_maximum; + else if (pstate->prefetch_target > 0) + pstate->prefetch_target *= 2; + else + pstate->prefetch_target++; + SpinLockRelease(&pstate->mutex); + } +#endif /* USE_PREFETCH */ +} + + +/* + * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target + */ +static inline void +BitmapPrefetch(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + BitmapHeapIterator *prefetch_iterator = scan->rs_base.rs_pf_bhs_iterator; + + if (pstate == NULL) + { + if (prefetch_iterator) + { + while (scan->prefetch_pages < scan->prefetch_target) + { + TBMIterateResult tbmpre; + bool skip_fetch; + + bhs_iterate(prefetch_iterator, &tbmpre); + + if (!BlockNumberIsValid(tbmpre.blockno)) + { + /* No more pages to prefetch */ + bhs_end_iterate(prefetch_iterator); + scan->rs_base.rs_pf_bhs_iterator = NULL; + break; + } + scan->prefetch_pages++; + scan->pfblockno = tbmpre.blockno; + + /* + * If we expect not to have to actually read this heap page, + * skip this prefetch call, but continue to run the prefetch + * logic normally. (Would it be better not to increment + * prefetch_pages?) + */ + skip_fetch = (!(scan->rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre.recheck && + VM_ALL_VISIBLE(scan->rs_base.rs_rd, + tbmpre.blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(scan->rs_base.rs_rd, MAIN_FORKNUM, tbmpre.blockno); + } + } + + return; + } + + if (pstate->prefetch_pages < pstate->prefetch_target) + { + if (prefetch_iterator) + { + while (1) + { + TBMIterateResult tbmpre; + bool do_prefetch = false; + bool skip_fetch; + + /* + * Recheck under the mutex. If some other process has already + * done enough prefetching then we need not to do anything. + */ + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages < pstate->prefetch_target) + { + pstate->prefetch_pages++; + do_prefetch = true; + } + SpinLockRelease(&pstate->mutex); + + if (!do_prefetch) + return; + + bhs_iterate(prefetch_iterator, &tbmpre); + if (!BlockNumberIsValid(tbmpre.blockno)) + { + /* No more pages to prefetch */ + bhs_end_iterate(prefetch_iterator); + scan->rs_base.rs_pf_bhs_iterator = NULL; + break; + } + + scan->pfblockno = tbmpre.blockno; + + /* As above, skip prefetch if we expect not to need page */ + skip_fetch = (!(scan->rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre.recheck && + VM_ALL_VISIBLE(scan->rs_base.rs_rd, + tbmpre.blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(scan->rs_base.rs_rd, MAIN_FORKNUM, tbmpre.blockno); + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_tuple(TableScanDesc scan, TupleTableSlot *slot) @@ -2390,6 +2622,36 @@ heapam_scan_bitmap_next_tuple(TableScanDesc scan, if (hscan->rs_cindex < 0 || hscan->rs_cindex >= hscan->rs_ntuples) return false; +#ifdef USE_PREFETCH + + /* + * Try to prefetch at least a few pages even before we get to the second + * page if we don't stop reading after the first tuple. + */ + if (!scan->bm_parallel) + { + if (hscan->prefetch_target < scan->prefetch_maximum) + hscan->prefetch_target++; + } + else if (scan->bm_parallel->prefetch_target < scan->prefetch_maximum) + { + /* take spinlock while updating shared state */ + SpinLockAcquire(&scan->bm_parallel->mutex); + if (scan->bm_parallel->prefetch_target < scan->prefetch_maximum) + scan->bm_parallel->prefetch_target++; + SpinLockRelease(&scan->bm_parallel->mutex); + } + + /* + * We issue prefetch requests *after* fetching the current page to try to + * avoid having prefetching interfere with the main I/O. Also, this should + * happen only when we have determined there is still something to do on + * the current page, else we may uselessly prefetch the same page we are + * just about to request for real. + */ + BitmapPrefetch(hscan); +#endif /* USE_PREFETCH */ + targoffset = hscan->rs_vistuples[hscan->rs_cindex]; page = BufferGetPage(hscan->rs_cbuf); lp = PageGetItemId(page, targoffset); diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index d61965a2761..187b288e688 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -51,10 +51,6 @@ static TupleTableSlot *BitmapHeapNext(BitmapHeapScanState *node); static inline void BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate); -static inline void BitmapAdjustPrefetchIterator(BitmapHeapScanState *node); -static inline void BitmapAdjustPrefetchTarget(BitmapHeapScanState *node); -static inline void BitmapPrefetch(BitmapHeapScanState *node, - TableScanDesc scan); static bool BitmapShouldInitializeSharedState(ParallelBitmapHeapState *pstate); static BitmapHeapIterator *bhs_begin_iterate(TIDBitmap *tbm, dsa_pointer shared_area, @@ -122,7 +118,6 @@ BitmapHeapNext(BitmapHeapScanState *node) TableScanDesc scan; TIDBitmap *tbm; TupleTableSlot *slot; - ParallelBitmapHeapState *pstate = node->pstate; dsa_area *dsa = node->ss.ps.state->es_query_dsa; /* @@ -142,7 +137,7 @@ BitmapHeapNext(BitmapHeapScanState *node) * prefetching. node->prefetch_pages tracks exactly how many pages ahead * the prefetch iterator is. Also, node->prefetch_target tracks the * desired prefetch distance, which starts small and increases up to the - * node->prefetch_maximum. This is to avoid doing a lot of prefetching in + * scan->prefetch_maximum. This is to avoid doing a lot of prefetching in * a scan that stops after a few tuples because of a LIMIT. */ if (!node->initialized) @@ -154,7 +149,16 @@ BitmapHeapNext(BitmapHeapScanState *node) bool init_shared_state = node->pstate ? BitmapShouldInitializeSharedState(node->pstate) : false; - if (!pstate || init_shared_state) + /* + * Maximum number of prefetches for the tablespace if configured, + * otherwise the current value of the effective_io_concurrency GUC. + */ + int pf_maximum = 0; +#ifdef USE_PREFETCH + pf_maximum = get_tablespace_io_concurrency(node->ss.ss_currentRelation->rd_rel->reltablespace); +#endif + + if (!node->pstate || init_shared_state) { tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); @@ -169,23 +173,16 @@ BitmapHeapNext(BitmapHeapScanState *node) * dsa_pointer of the iterator state which will be used by * multiple processes to iterate jointly. */ - pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); + node->pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (pf_maximum > 0) { - pstate->prefetch_iterator = + node->pstate->prefetch_iterator = tbm_prepare_shared_iterate(tbm); - - /* - * We don't need the mutex here as we haven't yet woke up - * others. - */ - pstate->prefetch_pages = 0; - pstate->prefetch_target = -1; } #endif /* We have initialized the shared state so wake up others. */ - BitmapDoneInitializingSharedState(pstate); + BitmapDoneInitializingSharedState(node->pstate); } } @@ -216,19 +213,19 @@ BitmapHeapNext(BitmapHeapScanState *node) extra_flags); } + scan->prefetch_maximum = pf_maximum; + scan->bm_parallel = node->pstate; + scan->rs_bhs_iterator = bhs_begin_iterate(tbm, - pstate ? pstate->tbmiterator : InvalidDsaPointer, + scan->bm_parallel ? scan->bm_parallel->tbmiterator : InvalidDsaPointer, dsa); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (scan->prefetch_maximum > 0) { - node->pf_iterator = bhs_begin_iterate(tbm, - pstate ? pstate->prefetch_iterator : InvalidDsaPointer, - dsa); - /* Only used for serial BHS */ - node->prefetch_pages = 0; - node->prefetch_target = -1; + scan->rs_pf_bhs_iterator = bhs_begin_iterate(tbm, + scan->bm_parallel ? scan->bm_parallel->prefetch_iterator : InvalidDsaPointer, + dsa); } #endif /* USE_PREFETCH */ @@ -243,36 +240,6 @@ BitmapHeapNext(BitmapHeapScanState *node) { CHECK_FOR_INTERRUPTS(); -#ifdef USE_PREFETCH - - /* - * Try to prefetch at least a few pages even before we get to the - * second page if we don't stop reading after the first tuple. - */ - if (!pstate) - { - if (node->prefetch_target < node->prefetch_maximum) - node->prefetch_target++; - } - else if (pstate->prefetch_target < node->prefetch_maximum) - { - /* take spinlock while updating shared state */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target < node->prefetch_maximum) - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ - - /* - * We issue prefetch requests *after* fetching the current page to - * try to avoid having prefetching interfere with the main I/O. - * Also, this should happen only when we have determined there is - * still something to do on the current page, else we may - * uselessly prefetch the same page we are just about to request - * for real. - */ - BitmapPrefetch(node, scan); /* * If we are using lossy info, we have to recheck the qual @@ -296,23 +263,9 @@ BitmapHeapNext(BitmapHeapScanState *node) new_page: - BitmapAdjustPrefetchIterator(node); - - if (!table_scan_bitmap_next_block(scan, &node->recheck, &node->blockno, + if (!table_scan_bitmap_next_block(scan, &node->recheck, &scan->blockno, &node->lossy_pages, &node->exact_pages)) break; - - /* - * If serial, we can error out if the the prefetch block doesn't stay - * ahead of the current block. - */ - if (node->pstate == NULL && - node->pf_iterator && - node->pfblockno < node->blockno) - elog(ERROR, "prefetch and main iterators are out of sync"); - - /* Adjust the prefetch target */ - BitmapAdjustPrefetchTarget(node); } /* @@ -336,219 +289,6 @@ BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate) ConditionVariableBroadcast(&pstate->cv); } -/* - * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator - * - * We keep track of how far the prefetch iterator is ahead of the main - * iterator in prefetch_pages. For each block the main iterator returns, we - * decrement prefetch_pages. - */ -static inline void -BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - BitmapHeapIterator *prefetch_iterator = node->pf_iterator; - TBMIterateResult tbmpre; - - if (pstate == NULL) - { - if (node->prefetch_pages > 0) - { - /* The main iterator has closed the distance by one page */ - node->prefetch_pages--; - } - else if (prefetch_iterator) - { - /* Do not let the prefetch iterator get behind the main one */ - bhs_iterate(prefetch_iterator, &tbmpre); - node->pfblockno = tbmpre.blockno; - } - return; - } - - /* - * Adjusting the prefetch iterator before invoking - * table_scan_bitmap_next_block() keeps prefetch distance higher across - * the parallel workers. - */ - if (node->prefetch_maximum > 0) - { - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages > 0) - { - pstate->prefetch_pages--; - SpinLockRelease(&pstate->mutex); - } - else - { - /* Release the mutex before iterating */ - SpinLockRelease(&pstate->mutex); - - /* - * In case of shared mode, we can not ensure that the current - * blockno of the main iterator and that of the prefetch iterator - * are same. It's possible that whatever blockno we are - * prefetching will be processed by another process. Therefore, - * we don't validate the blockno here as we do in non-parallel - * case. - */ - if (prefetch_iterator) - { - bhs_iterate(prefetch_iterator, &tbmpre); - node->pfblockno = tbmpre.blockno; - } - } - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapAdjustPrefetchTarget - Adjust the prefetch target - * - * Increase prefetch target if it's not yet at the max. Note that - * we will increase it to zero after fetching the very first - * page/tuple, then to one after the second tuple is fetched, then - * it doubles as later pages are fetched. - */ -static inline void -BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - - if (pstate == NULL) - { - if (node->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (node->prefetch_target >= node->prefetch_maximum / 2) - node->prefetch_target = node->prefetch_maximum; - else if (node->prefetch_target > 0) - node->prefetch_target *= 2; - else - node->prefetch_target++; - return; - } - - /* Do an unlocked check first to save spinlock acquisitions. */ - if (pstate->prefetch_target < node->prefetch_maximum) - { - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (pstate->prefetch_target >= node->prefetch_maximum / 2) - pstate->prefetch_target = node->prefetch_maximum; - else if (pstate->prefetch_target > 0) - pstate->prefetch_target *= 2; - else - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target - */ -static inline void -BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - BitmapHeapIterator *prefetch_iterator = node->pf_iterator; - - if (pstate == NULL) - { - if (prefetch_iterator) - { - while (node->prefetch_pages < node->prefetch_target) - { - TBMIterateResult tbmpre; - bool skip_fetch; - - bhs_iterate(prefetch_iterator, &tbmpre); - - if (!BlockNumberIsValid(tbmpre.blockno)) - { - /* No more pages to prefetch */ - bhs_end_iterate(prefetch_iterator); - node->pf_iterator = NULL; - break; - } - node->prefetch_pages++; - node->pfblockno = tbmpre.blockno; - - /* - * If we expect not to have to actually read this heap page, - * skip this prefetch call, but continue to run the prefetch - * logic normally. (Would it be better not to increment - * prefetch_pages?) - */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre.recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre.blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre.blockno); - } - } - - return; - } - - if (pstate->prefetch_pages < pstate->prefetch_target) - { - if (prefetch_iterator) - { - while (1) - { - TBMIterateResult tbmpre; - bool do_prefetch = false; - bool skip_fetch; - - /* - * Recheck under the mutex. If some other process has already - * done enough prefetching then we need not to do anything. - */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages < pstate->prefetch_target) - { - pstate->prefetch_pages++; - do_prefetch = true; - } - SpinLockRelease(&pstate->mutex); - - if (!do_prefetch) - return; - - bhs_iterate(prefetch_iterator, &tbmpre); - if (!BlockNumberIsValid(tbmpre.blockno)) - { - /* No more pages to prefetch */ - bhs_end_iterate(prefetch_iterator); - node->pf_iterator = NULL; - break; - } - - node->pfblockno = tbmpre.blockno; - - /* As above, skip prefetch if we expect not to need page */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre.recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre.blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre.blockno); - } - } - } -#endif /* USE_PREFETCH */ -} - /* * BitmapHeapRecheck -- access method routine to recheck a tuple in EvalPlanQual */ @@ -594,22 +334,12 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) if (node->ss.ss_currentScanDesc) table_rescan(node->ss.ss_currentScanDesc, NULL); - /* release bitmaps and buffers if any */ - if (node->pf_iterator) - { - bhs_end_iterate(node->pf_iterator); - node->pf_iterator = NULL; - } + /* release bitmaps if any */ if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); node->tbm = NULL; node->initialized = false; - node->pvmbuffer = InvalidBuffer; node->recheck = true; - node->blockno = InvalidBlockNumber; - node->pfblockno = InvalidBlockNumber; ExecScanReScan(&node->ss); @@ -648,14 +378,10 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) table_endscan(scanDesc); /* - * release bitmaps and buffers if any + * release bitmaps if any */ - if (node->pf_iterator) - bhs_end_iterate(node->pf_iterator); if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); } /* ---------------------------------------------------------------- @@ -688,17 +414,11 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ps.ExecProcNode = ExecBitmapHeapScan; scanstate->tbm = NULL; - scanstate->pvmbuffer = InvalidBuffer; scanstate->exact_pages = 0; scanstate->lossy_pages = 0; - scanstate->pf_iterator = NULL; - scanstate->prefetch_pages = 0; - scanstate->prefetch_target = 0; scanstate->initialized = false; scanstate->pstate = NULL; scanstate->recheck = true; - scanstate->blockno = InvalidBlockNumber; - scanstate->pfblockno = InvalidBlockNumber; /* * Miscellaneous initialization @@ -738,13 +458,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->bitmapqualorig = ExecInitQual(node->bitmapqualorig, (PlanState *) scanstate); - /* - * Maximum number of prefetches for the tablespace if configured, - * otherwise the current value of the effective_io_concurrency GUC. - */ - scanstate->prefetch_maximum = - get_tablespace_io_concurrency(currentRelation->rd_rel->reltablespace); - scanstate->ss.ss_currentRelation = currentRelation; /* @@ -828,7 +541,7 @@ ExecBitmapHeapInitializeDSM(BitmapHeapScanState *node, /* Initialize the mutex */ SpinLockInit(&pstate->mutex); pstate->prefetch_pages = 0; - pstate->prefetch_target = 0; + pstate->prefetch_target = -1; pstate->state = BM_INITIAL; ConditionVariableInit(&pstate->cv); diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 49f212eb9a6..accbc1749cd 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -82,6 +82,23 @@ typedef struct HeapScanDescData Buffer rs_vmbuffer; int rs_empty_tuples_pending; + /* + * These fields only used for prefetching in bitmap table scans + */ + + /* buffer for visibility-map lookups of prefetched pages */ + Buffer pvmbuffer; + + /* + * These fields only used in serial BHS + */ + /* Current target for prefetch distance */ + int prefetch_target; + /* # pages prefetch iterator is ahead of current */ + int prefetch_pages; + /* used to validate prefetch block stays ahead of current block */ + BlockNumber pfblockno; + /* these fields only used in page-at-a-time mode and for bitmap scans */ int rs_cindex; /* current tuple's index in vistuples */ int rs_ntuples; /* number of visible tuples on page */ diff --git a/src/include/access/relscan.h b/src/include/access/relscan.h index fb22f305bf6..7938b741d66 100644 --- a/src/include/access/relscan.h +++ b/src/include/access/relscan.h @@ -26,6 +26,7 @@ struct ParallelTableScanDescData; struct BitmapHeapIterator; +struct ParallelBitmapHeapState; /* * Generic descriptor for table scans. This is the base-class for table scans, @@ -45,6 +46,13 @@ typedef struct TableScanDescData /* Only used for Bitmap table scans */ struct BitmapHeapIterator *rs_bhs_iterator; + struct BitmapHeapIterator *rs_pf_bhs_iterator; + + /* maximum value for prefetch_target */ + int prefetch_maximum; + struct ParallelBitmapHeapState *bm_parallel; + /* used to validate BHS prefetch and current block stay in sync */ + BlockNumber blockno; /* * Information about type and behaviour of the scan, a bitmask of members diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index a06ed271eb6..a30c0412901 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -807,17 +807,6 @@ typedef struct TableAmRoutine * lossy_pages is incremented if the block's representation in the bitmap * is lossy, otherwise, exact_pages is incremented. * - * XXX: Currently this may only be implemented if the AM uses md.c as its - * storage manager, and uses ItemPointer->ip_blkid in a manner that maps - * blockids directly to the underlying storage. nodeBitmapHeapscan.c - * performs prefetching directly using that interface. This probably - * needs to be rectified at a later point. - * - * XXX: Currently this may only be implemented if the AM uses the - * visibilitymap, as nodeBitmapHeapscan.c unconditionally accesses it to - * perform prefetching. This probably needs to be rectified at a later - * point. - * * Optional callback, but either both scan_bitmap_next_block and * scan_bitmap_next_tuple need to exist, or neither. */ @@ -967,6 +956,9 @@ table_beginscan_bm(Relation rel, Snapshot snapshot, result = rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); result->rs_bhs_iterator = NULL; + result->rs_pf_bhs_iterator = NULL; + result->prefetch_maximum = 0; + result->bm_parallel = NULL; return result; } @@ -1018,6 +1010,12 @@ table_endscan(TableScanDesc scan) { bhs_end_iterate(scan->rs_bhs_iterator); scan->rs_bhs_iterator = NULL; + + if (scan->rs_pf_bhs_iterator) + { + bhs_end_iterate(scan->rs_pf_bhs_iterator); + scan->rs_pf_bhs_iterator = NULL; + } } scan->rs_rd->rd_tableam->scan_end(scan); @@ -1034,6 +1032,12 @@ table_rescan(TableScanDesc scan, { bhs_end_iterate(scan->rs_bhs_iterator); scan->rs_bhs_iterator = NULL; + + if (scan->rs_pf_bhs_iterator) + { + bhs_end_iterate(scan->rs_pf_bhs_iterator); + scan->rs_pf_bhs_iterator = NULL; + } } scan->rs_rd->rd_tableam->scan_rescan(scan, key, false, false, false, false); diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index cf8b4995f0d..592215d5ee7 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1794,18 +1794,11 @@ struct BitmapHeapIterator; * * bitmapqualorig execution state for bitmapqualorig expressions * tbm bitmap obtained from child index scan(s) - * pvmbuffer buffer for visibility-map lookups of prefetched pages * exact_pages total number of exact pages retrieved * lossy_pages total number of lossy pages retrieved - * pf_iterator for prefetching ahead of current page - * prefetch_pages # pages prefetch iterator is ahead of current - * prefetch_target current target prefetch distance - * prefetch_maximum maximum value for prefetch_target * initialized is node is ready to iterate * pstate shared state for parallel bitmap scan * recheck do current page's tuples need recheck - * blockno used to validate pf and current block in sync - * pfblockno used to validate pf stays ahead of current block * ---------------- */ typedef struct BitmapHeapScanState @@ -1813,18 +1806,11 @@ typedef struct BitmapHeapScanState ScanState ss; /* its first field is NodeTag */ ExprState *bitmapqualorig; TIDBitmap *tbm; - Buffer pvmbuffer; long exact_pages; long lossy_pages; - int prefetch_pages; - int prefetch_target; - int prefetch_maximum; bool initialized; - struct BitmapHeapIterator *pf_iterator; ParallelBitmapHeapState *pstate; bool recheck; - BlockNumber blockno; - BlockNumber pfblockno; } BitmapHeapScanState; /* ---------------- -- 2.40.1 --6jpz2j246qmht4bt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0015-Remove-table_scan_bitmap_next_block.patch" ^ permalink raw reply [nested|flat] 10+ messages in thread
* [PATCH v8 13/17] Push BitmapHeapScan prefetch code into heapam.c @ 2024-03-22 13:42 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 10+ messages in thread From: Melanie Plageman @ 2024-03-22 13:42 UTC (permalink / raw) In preparation for transitioning to using the streaming read API for prefetching [1], move all of the BitmapHeapScanState members related to prefetching and the functions for accessing them into the HeapScanDescData and TableScanDescData. Members that still need to be accessed in BitmapHeapNext() could not be moved into heap AM-specific code. Specifically, parallel iterator setup requires several components which seem odd to pass to the table AM API. [1] https://www.postgresql.org/message-id/flat/CA%2BhUKGJkOiOCa%2Bmag4BF%2BzHo7qo%3Do9CFheB8%3Dg6uT5TUm2... --- src/backend/access/heap/heapam.c | 26 ++ src/backend/access/heap/heapam_handler.c | 268 +++++++++++++++ src/backend/executor/nodeBitmapHeapscan.c | 397 +++------------------- src/include/access/heapam.h | 12 + src/include/access/relscan.h | 11 + src/include/access/tableam.h | 38 ++- src/include/nodes/execnodes.h | 16 - 7 files changed, 388 insertions(+), 380 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index e7bed84f75..c12563a188 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -951,8 +951,16 @@ heap_beginscan(Relation relation, Snapshot snapshot, scan->rs_base.rs_flags = flags; scan->rs_base.rs_parallel = parallel_scan; scan->rs_strategy = NULL; /* set in initscan */ + + scan->rs_base.blockno = InvalidBlockNumber; + scan->rs_vmbuffer = InvalidBuffer; scan->rs_empty_tuples_pending = 0; + scan->pvmbuffer = InvalidBuffer; + + scan->pfblockno = InvalidBlockNumber; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; /* * Disable page-at-a-time mode if it's not a MVCC-safe snapshot. @@ -1035,6 +1043,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, scan->rs_base.rs_flags &= ~SO_ALLOW_PAGEMODE; } + scan->rs_base.blockno = InvalidBlockNumber; + + scan->pfblockno = InvalidBlockNumber; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; + /* * unpin scan buffers */ @@ -1047,6 +1061,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, scan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(scan->pvmbuffer)) + { + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + } + /* * reinitialize scan descriptor */ @@ -1072,6 +1092,12 @@ heap_endscan(TableScanDesc sscan) scan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(scan->pvmbuffer)) + { + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + } + /* * decrement relation reference count and free scan descriptor storage */ diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index adfc77684a..efd2784e03 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -55,6 +55,9 @@ static bool SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, OffsetNumber tupoffset); static BlockNumber heapam_scan_get_blocks_done(HeapScanDesc hscan); +static inline void BitmapAdjustPrefetchIterator(HeapScanDesc scan); +static inline void BitmapAdjustPrefetchTarget(HeapScanDesc scan); +static inline void BitmapPrefetch(HeapScanDesc scan); static const TableAmRoutine heapam_methods; @@ -2112,6 +2115,76 @@ heapam_estimate_rel_size(Relation rel, int32 *attr_widths, * ------------------------------------------------------------------------ */ +/* + * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator + * + * We keep track of how far the prefetch iterator is ahead of the main + * iterator in prefetch_pages. For each block the main iterator returns, we + * decrement prefetch_pages. + */ +static inline void +BitmapAdjustPrefetchIterator(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + TBMIterateResult tbmpre; + + if (pstate == NULL) + { + TBMIterator *prefetch_iterator = scan->rs_base.pf_tbmiterator; + + if (scan->prefetch_pages > 0) + { + /* The main iterator has closed the distance by one page */ + scan->prefetch_pages--; + } + else if (prefetch_iterator) + { + /* Do not let the prefetch iterator get behind the main one */ + tbm_iterate(prefetch_iterator, &tbmpre); + scan->pfblockno = tbmpre.blockno; + } + return; + } + + /* + * Adjusting the prefetch iterator before invoking + * table_scan_bitmap_next_block() keeps prefetch distance higher across + * the parallel workers. + */ + if (scan->rs_base.prefetch_maximum > 0) + { + TBMSharedIterator *prefetch_iterator = scan->rs_base.pf_shared_tbmiterator; + + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages > 0) + { + pstate->prefetch_pages--; + SpinLockRelease(&pstate->mutex); + } + else + { + /* Release the mutex before iterating */ + SpinLockRelease(&pstate->mutex); + + /* + * In case of shared mode, we can not ensure that the current + * blockno of the main iterator and that of the prefetch iterator + * are same. It's possible that whatever blockno we are + * prefetching will be processed by another process. Therefore, + * we don't validate the blockno here as we do in non-parallel + * case. + */ + if (prefetch_iterator) + { + tbm_shared_iterate(prefetch_iterator, &tbmpre); + scan->pfblockno = tbmpre.blockno; + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_block(TableScanDesc scan, bool *recheck, BlockNumber *blockno, @@ -2130,6 +2203,8 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, *blockno = InvalidBlockNumber; *recheck = true; + BitmapAdjustPrefetchIterator(hscan); + do { CHECK_FOR_INTERRUPTS(); @@ -2273,6 +2348,18 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, else (*exact_pages)++; + /* + * If serial, we can error out if the the prefetch block doesn't stay + * ahead of the current block. + */ + if (scan->bm_parallel == NULL && + scan->pf_tbmiterator && + hscan->pfblockno > hscan->rs_base.blockno) + elog(ERROR, "prefetch and main iterators are out of sync"); + + /* Adjust the prefetch target */ + BitmapAdjustPrefetchTarget(hscan); + /* * Return true to indicate that a valid block was found and the bitmap is * not exhausted. If there are no visible tuples on this page, @@ -2283,6 +2370,157 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, return true; } +/* + * BitmapAdjustPrefetchTarget - Adjust the prefetch target + * + * Increase prefetch target if it's not yet at the max. Note that + * we will increase it to zero after fetching the very first + * page/tuple, then to one after the second tuple is fetched, then + * it doubles as later pages are fetched. + */ +static inline void +BitmapAdjustPrefetchTarget(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + int prefetch_maximum = scan->rs_base.prefetch_maximum; + + if (pstate == NULL) + { + if (scan->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (scan->prefetch_target >= prefetch_maximum / 2) + scan->prefetch_target = prefetch_maximum; + else if (scan->prefetch_target > 0) + scan->prefetch_target *= 2; + else + scan->prefetch_target++; + return; + } + + /* Do an unlocked check first to save spinlock acquisitions. */ + if (pstate->prefetch_target < prefetch_maximum) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (pstate->prefetch_target >= prefetch_maximum / 2) + pstate->prefetch_target = prefetch_maximum; + else if (pstate->prefetch_target > 0) + pstate->prefetch_target *= 2; + else + pstate->prefetch_target++; + SpinLockRelease(&pstate->mutex); + } +#endif /* USE_PREFETCH */ +} + + +/* + * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target + */ +static inline void +BitmapPrefetch(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + + if (pstate == NULL) + { + TBMIterator *prefetch_iterator = scan->rs_base.pf_tbmiterator; + + if (prefetch_iterator) + { + while (scan->prefetch_pages < scan->prefetch_target) + { + TBMIterateResult tbmpre; + bool skip_fetch; + + tbm_iterate(prefetch_iterator, &tbmpre); + + if (!BlockNumberIsValid(tbmpre.blockno)) + { + /* No more pages to prefetch */ + tbm_end_iterate(prefetch_iterator); + scan->rs_base.pf_tbmiterator = NULL; + break; + } + scan->prefetch_pages++; + scan->pfblockno = tbmpre.blockno; + + /* + * If we expect not to have to actually read this heap page, + * skip this prefetch call, but continue to run the prefetch + * logic normally. (Would it be better not to increment + * prefetch_pages?) + */ + skip_fetch = (!(scan->rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre.recheck && + VM_ALL_VISIBLE(scan->rs_base.rs_rd, + tbmpre.blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(scan->rs_base.rs_rd, MAIN_FORKNUM, tbmpre.blockno); + } + } + + return; + } + + if (pstate->prefetch_pages < pstate->prefetch_target) + { + TBMSharedIterator *prefetch_iterator = scan->rs_base.pf_shared_tbmiterator; + + if (prefetch_iterator) + { + while (1) + { + TBMIterateResult tbmpre; + bool do_prefetch = false; + bool skip_fetch; + + /* + * Recheck under the mutex. If some other process has already + * done enough prefetching then we need not to do anything. + */ + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages < pstate->prefetch_target) + { + pstate->prefetch_pages++; + do_prefetch = true; + } + SpinLockRelease(&pstate->mutex); + + if (!do_prefetch) + return; + + tbm_shared_iterate(prefetch_iterator, &tbmpre); + if (!BlockNumberIsValid(tbmpre.blockno)) + { + /* No more pages to prefetch */ + tbm_end_shared_iterate(prefetch_iterator); + scan->rs_base.pf_shared_tbmiterator = NULL; + break; + } + + scan->pfblockno = tbmpre.blockno; + + /* As above, skip prefetch if we expect not to need page */ + skip_fetch = (!(scan->rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre.recheck && + VM_ALL_VISIBLE(scan->rs_base.rs_rd, + tbmpre.blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(scan->rs_base.rs_rd, MAIN_FORKNUM, tbmpre.blockno); + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_tuple(TableScanDesc scan, TupleTableSlot *slot) @@ -2308,6 +2546,36 @@ heapam_scan_bitmap_next_tuple(TableScanDesc scan, if (hscan->rs_cindex < 0 || hscan->rs_cindex >= hscan->rs_ntuples) return false; +#ifdef USE_PREFETCH + + /* + * Try to prefetch at least a few pages even before we get to the second + * page if we don't stop reading after the first tuple. + */ + if (!scan->bm_parallel) + { + if (hscan->prefetch_target < scan->prefetch_maximum) + hscan->prefetch_target++; + } + else if (scan->bm_parallel->prefetch_target < scan->prefetch_maximum) + { + /* take spinlock while updating shared state */ + SpinLockAcquire(&scan->bm_parallel->mutex); + if (scan->bm_parallel->prefetch_target < scan->prefetch_maximum) + scan->bm_parallel->prefetch_target++; + SpinLockRelease(&scan->bm_parallel->mutex); + } + + /* + * We issue prefetch requests *after* fetching the current page to try to + * avoid having prefetching interfere with the main I/O. Also, this should + * happen only when we have determined there is still something to do on + * the current page, else we may uselessly prefetch the same page we are + * just about to request for real. + */ + BitmapPrefetch(hscan); +#endif /* USE_PREFETCH */ + targoffset = hscan->rs_vistuples[hscan->rs_cindex]; page = BufferGetPage(hscan->rs_cbuf); lp = PageGetItemId(page, targoffset); diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index 51c4360205..f241f4cb2c 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -51,10 +51,6 @@ static TupleTableSlot *BitmapHeapNext(BitmapHeapScanState *node); static inline void BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate); -static inline void BitmapAdjustPrefetchIterator(BitmapHeapScanState *node); -static inline void BitmapAdjustPrefetchTarget(BitmapHeapScanState *node); -static inline void BitmapPrefetch(BitmapHeapScanState *node, - TableScanDesc scan); static bool BitmapShouldInitializeSharedState(ParallelBitmapHeapState *pstate); @@ -71,7 +67,6 @@ BitmapHeapNext(BitmapHeapScanState *node) TableScanDesc scan; TIDBitmap *tbm; TupleTableSlot *slot; - ParallelBitmapHeapState *pstate = node->pstate; dsa_area *dsa = node->ss.ps.state->es_query_dsa; /* @@ -91,83 +86,53 @@ BitmapHeapNext(BitmapHeapScanState *node) * prefetching. node->prefetch_pages tracks exactly how many pages ahead * the prefetch iterator is. Also, node->prefetch_target tracks the * desired prefetch distance, which starts small and increases up to the - * node->prefetch_maximum. This is to avoid doing a lot of prefetching in + * scan->prefetch_maximum. This is to avoid doing a lot of prefetching in * a scan that stops after a few tuples because of a LIMIT. */ if (!node->initialized) { - TBMIterator *tbmiterator = NULL; - TBMSharedIterator *shared_tbmiterator = NULL; + /* + * The leader will immediately come out of the function, but others + * will be blocked until leader populates the TBM and wakes them up. + */ + bool init_shared_state = node->pstate ? + BitmapShouldInitializeSharedState(node->pstate) : false; + + /* + * Maximum number of prefetches for the tablespace if configured, + * otherwise the current value of the effective_io_concurrency GUC. + */ + int pf_maximum = 0; +#ifdef USE_PREFETCH + pf_maximum = get_tablespace_io_concurrency(node->ss.ss_currentRelation->rd_rel->reltablespace); +#endif - if (!pstate) + if (!node->pstate || init_shared_state) { tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); if (!tbm || !IsA(tbm, TIDBitmap)) elog(ERROR, "unrecognized result from subplan"); - node->tbm = tbm; - tbmiterator = tbm_begin_iterate(tbm); -#ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (init_shared_state) { - node->prefetch_iterator = tbm_begin_iterate(tbm); - node->prefetch_pages = 0; - node->prefetch_target = -1; - } -#endif /* USE_PREFETCH */ - } - else - { - /* - * The leader will immediately come out of the function, but - * others will be blocked until leader populates the TBM and wakes - * them up. - */ - if (BitmapShouldInitializeSharedState(pstate)) - { - tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); - if (!tbm || !IsA(tbm, TIDBitmap)) - elog(ERROR, "unrecognized result from subplan"); - - node->tbm = tbm; - /* * Prepare to iterate over the TBM. This will return the * dsa_pointer of the iterator state which will be used by * multiple processes to iterate jointly. */ - pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); + node->pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (pf_maximum > 0) { - pstate->prefetch_iterator = + node->pstate->prefetch_iterator = tbm_prepare_shared_iterate(tbm); - - /* - * We don't need the mutex here as we haven't yet woke up - * others. - */ - pstate->prefetch_pages = 0; - pstate->prefetch_target = -1; } #endif - /* We have initialized the shared state so wake up others. */ - BitmapDoneInitializingSharedState(pstate); - } - - /* Allocate a private iterator and attach the shared state to it */ - shared_tbmiterator = tbm_attach_shared_iterate(dsa, pstate->tbmiterator); - -#ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) - { - node->shared_prefetch_iterator = - tbm_attach_shared_iterate(dsa, pstate->prefetch_iterator); + BitmapDoneInitializingSharedState(node->pstate); } -#endif /* USE_PREFETCH */ } /* @@ -197,8 +162,26 @@ BitmapHeapNext(BitmapHeapScanState *node) extra_flags); } - scan->tbmiterator = tbmiterator; - scan->shared_tbmiterator = shared_tbmiterator; + scan->prefetch_maximum = pf_maximum; + scan->bm_parallel = node->pstate; + + if (!scan->bm_parallel) + scan->tbmiterator = tbm_begin_iterate(tbm); + else + /* Allocate a private iterator and attach the shared state to it */ + scan->shared_tbmiterator = tbm_attach_shared_iterate(dsa, scan->bm_parallel->tbmiterator); + +#ifdef USE_PREFETCH + if (scan->prefetch_maximum > 0) + { + if (!scan->bm_parallel) + scan->pf_tbmiterator = tbm_begin_iterate(tbm); + else + scan->pf_shared_tbmiterator = + tbm_attach_shared_iterate(dsa, scan->bm_parallel->prefetch_iterator); + } +#endif /* USE_PREFETCH */ + node->initialized = true; @@ -211,36 +194,6 @@ BitmapHeapNext(BitmapHeapScanState *node) { CHECK_FOR_INTERRUPTS(); -#ifdef USE_PREFETCH - - /* - * Try to prefetch at least a few pages even before we get to the - * second page if we don't stop reading after the first tuple. - */ - if (!pstate) - { - if (node->prefetch_target < node->prefetch_maximum) - node->prefetch_target++; - } - else if (pstate->prefetch_target < node->prefetch_maximum) - { - /* take spinlock while updating shared state */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target < node->prefetch_maximum) - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ - - /* - * We issue prefetch requests *after* fetching the current page to - * try to avoid having prefetching interfere with the main I/O. - * Also, this should happen only when we have determined there is - * still something to do on the current page, else we may - * uselessly prefetch the same page we are just about to request - * for real. - */ - BitmapPrefetch(node, scan); /* * If we are using lossy info, we have to recheck the qual @@ -264,23 +217,9 @@ BitmapHeapNext(BitmapHeapScanState *node) new_page: - BitmapAdjustPrefetchIterator(node); - - if (!table_scan_bitmap_next_block(scan, &node->recheck, &node->blockno, + if (!table_scan_bitmap_next_block(scan, &node->recheck, &scan->blockno, &node->lossy_pages, &node->exact_pages)) break; - - /* - * If serial, we can error out if the the prefetch block doesn't stay - * ahead of the current block. - */ - if (node->pstate == NULL && - node->prefetch_iterator && - node->pfblockno > node->blockno) - elog(ERROR, "prefetch and main iterators are out of sync"); - - /* Adjust the prefetch target */ - BitmapAdjustPrefetchTarget(node); } /* @@ -304,224 +243,7 @@ BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate) ConditionVariableBroadcast(&pstate->cv); } -/* - * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator - * - * We keep track of how far the prefetch iterator is ahead of the main - * iterator in prefetch_pages. For each block the main iterator returns, we - * decrement prefetch_pages. - */ -static inline void -BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - TBMIterateResult tbmpre; - - if (pstate == NULL) - { - TBMIterator *prefetch_iterator = node->prefetch_iterator; - - if (node->prefetch_pages > 0) - { - /* The main iterator has closed the distance by one page */ - node->prefetch_pages--; - } - else if (prefetch_iterator) - { - /* Do not let the prefetch iterator get behind the main one */ - tbm_iterate(prefetch_iterator, &tbmpre); - node->pfblockno = tbmpre.blockno; - } - return; - } - - /* - * Adjusting the prefetch iterator before invoking - * table_scan_bitmap_next_block() keeps prefetch distance higher across - * the parallel workers. - */ - if (node->prefetch_maximum > 0) - { - TBMSharedIterator *prefetch_iterator = node->shared_prefetch_iterator; - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages > 0) - { - pstate->prefetch_pages--; - SpinLockRelease(&pstate->mutex); - } - else - { - /* Release the mutex before iterating */ - SpinLockRelease(&pstate->mutex); - - /* - * In case of shared mode, we can not ensure that the current - * blockno of the main iterator and that of the prefetch iterator - * are same. It's possible that whatever blockno we are - * prefetching will be processed by another process. Therefore, - * we don't validate the blockno here as we do in non-parallel - * case. - */ - if (prefetch_iterator) - { - tbm_shared_iterate(prefetch_iterator, &tbmpre); - node->pfblockno = tbmpre.blockno; - } - } - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapAdjustPrefetchTarget - Adjust the prefetch target - * - * Increase prefetch target if it's not yet at the max. Note that - * we will increase it to zero after fetching the very first - * page/tuple, then to one after the second tuple is fetched, then - * it doubles as later pages are fetched. - */ -static inline void -BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - - if (pstate == NULL) - { - if (node->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (node->prefetch_target >= node->prefetch_maximum / 2) - node->prefetch_target = node->prefetch_maximum; - else if (node->prefetch_target > 0) - node->prefetch_target *= 2; - else - node->prefetch_target++; - return; - } - - /* Do an unlocked check first to save spinlock acquisitions. */ - if (pstate->prefetch_target < node->prefetch_maximum) - { - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (pstate->prefetch_target >= node->prefetch_maximum / 2) - pstate->prefetch_target = node->prefetch_maximum; - else if (pstate->prefetch_target > 0) - pstate->prefetch_target *= 2; - else - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target - */ -static inline void -BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - - if (pstate == NULL) - { - TBMIterator *prefetch_iterator = node->prefetch_iterator; - - if (prefetch_iterator) - { - while (node->prefetch_pages < node->prefetch_target) - { - TBMIterateResult tbmpre; - bool skip_fetch; - - tbm_iterate(prefetch_iterator, &tbmpre); - - if (!BlockNumberIsValid(tbmpre.blockno)) - { - /* No more pages to prefetch */ - tbm_end_iterate(prefetch_iterator); - node->prefetch_iterator = NULL; - break; - } - node->prefetch_pages++; - node->pfblockno = tbmpre.blockno; - - /* - * If we expect not to have to actually read this heap page, - * skip this prefetch call, but continue to run the prefetch - * logic normally. (Would it be better not to increment - * prefetch_pages?) - */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre.recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre.blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre.blockno); - } - } - - return; - } - - if (pstate->prefetch_pages < pstate->prefetch_target) - { - TBMSharedIterator *prefetch_iterator = node->shared_prefetch_iterator; - - if (prefetch_iterator) - { - while (1) - { - TBMIterateResult tbmpre; - bool do_prefetch = false; - bool skip_fetch; - - /* - * Recheck under the mutex. If some other process has already - * done enough prefetching then we need not to do anything. - */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages < pstate->prefetch_target) - { - pstate->prefetch_pages++; - do_prefetch = true; - } - SpinLockRelease(&pstate->mutex); - - if (!do_prefetch) - return; - - tbm_shared_iterate(prefetch_iterator, &tbmpre); - if (!BlockNumberIsValid(tbmpre.blockno)) - { - /* No more pages to prefetch */ - tbm_end_shared_iterate(prefetch_iterator); - node->shared_prefetch_iterator = NULL; - break; - } - - node->pfblockno = tbmpre.blockno; - - /* As above, skip prefetch if we expect not to need page */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre.recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre.blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre.blockno); - } - } - } -#endif /* USE_PREFETCH */ -} /* * BitmapHeapRecheck -- access method routine to recheck a tuple in EvalPlanQual @@ -569,22 +291,11 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ - if (node->prefetch_iterator) - tbm_end_iterate(node->prefetch_iterator); - if (node->shared_prefetch_iterator) - tbm_end_shared_iterate(node->shared_prefetch_iterator); if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); node->tbm = NULL; - node->prefetch_iterator = NULL; node->initialized = false; - node->shared_prefetch_iterator = NULL; - node->pvmbuffer = InvalidBuffer; node->recheck = true; - node->blockno = InvalidBlockNumber; - node->pfblockno = InvalidBlockNumber; ExecScanReScan(&node->ss); @@ -625,14 +336,8 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * release bitmaps and buffers if any */ - if (node->prefetch_iterator) - tbm_end_iterate(node->prefetch_iterator); if (node->tbm) tbm_free(node->tbm); - if (node->shared_prefetch_iterator) - tbm_end_shared_iterate(node->shared_prefetch_iterator); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); } /* ---------------------------------------------------------------- @@ -665,18 +370,11 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ps.ExecProcNode = ExecBitmapHeapScan; scanstate->tbm = NULL; - scanstate->pvmbuffer = InvalidBuffer; scanstate->exact_pages = 0; scanstate->lossy_pages = 0; - scanstate->prefetch_iterator = NULL; - scanstate->prefetch_pages = 0; - scanstate->prefetch_target = 0; scanstate->initialized = false; - scanstate->shared_prefetch_iterator = NULL; scanstate->pstate = NULL; scanstate->recheck = true; - scanstate->blockno = InvalidBlockNumber; - scanstate->pfblockno = InvalidBlockNumber; /* * Miscellaneous initialization @@ -716,13 +414,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->bitmapqualorig = ExecInitQual(node->bitmapqualorig, (PlanState *) scanstate); - /* - * Maximum number of prefetches for the tablespace if configured, - * otherwise the current value of the effective_io_concurrency GUC. - */ - scanstate->prefetch_maximum = - get_tablespace_io_concurrency(currentRelation->rd_rel->reltablespace); - scanstate->ss.ss_currentRelation = currentRelation; /* @@ -806,7 +497,7 @@ ExecBitmapHeapInitializeDSM(BitmapHeapScanState *node, /* Initialize the mutex */ SpinLockInit(&pstate->mutex); pstate->prefetch_pages = 0; - pstate->prefetch_target = 0; + pstate->prefetch_target = -1; pstate->state = BM_INITIAL; ConditionVariableInit(&pstate->cv); diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 3dfb19ec7d..22bdccc2a9 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -81,6 +81,18 @@ typedef struct HeapScanDescData */ Buffer rs_vmbuffer; int rs_empty_tuples_pending; + /* buffer for visibility-map lookups of prefetched pages */ + Buffer pvmbuffer; + + /* + * These fields only used for prefetching in bitmap table scans + */ + /* Current target for prefetch distance */ + int prefetch_target; + /* # pages prefetch iterator is ahead of current */ + int prefetch_pages; + /* used to validate prefetch block stays ahead of current block */ + BlockNumber pfblockno; /* these fields only used in page-at-a-time mode and for bitmap scans */ int rs_cindex; /* current tuple's index in vistuples */ diff --git a/src/include/access/relscan.h b/src/include/access/relscan.h index 92b829cebc..93168bd350 100644 --- a/src/include/access/relscan.h +++ b/src/include/access/relscan.h @@ -26,6 +26,7 @@ struct ParallelTableScanDescData; struct TBMIterator; struct TBMSharedIterator; +struct ParallelBitmapHeapState; /* * Generic descriptor for table scans. This is the base-class for table scans, @@ -46,6 +47,16 @@ typedef struct TableScanDescData /* Only used for Bitmap table scans */ struct TBMIterator *tbmiterator; struct TBMSharedIterator *shared_tbmiterator; + /* Prefetch iterators */ + struct TBMIterator *pf_tbmiterator; + struct TBMSharedIterator *pf_shared_tbmiterator; + + /* maximum value for prefetch_target */ + int prefetch_maximum; + struct ParallelBitmapHeapState *bm_parallel; + + /* used to validate prefetch and current block stay in sync */ + BlockNumber blockno; /* * Information about type and behaviour of the scan, a bitmask of members diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 1d4b79a73f..9cab4462d6 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -800,17 +800,6 @@ typedef struct TableAmRoutine * lossy_pages is incremented if the block's representation in the bitmap * is lossy, otherwise, exact_pages is incremented. * - * XXX: Currently this may only be implemented if the AM uses md.c as its - * storage manager, and uses ItemPointer->ip_blkid in a manner that maps - * blockids directly to the underlying storage. nodeBitmapHeapscan.c - * performs prefetching directly using that interface. This probably - * needs to be rectified at a later point. - * - * XXX: Currently this may only be implemented if the AM uses the - * visibilitymap, as nodeBitmapHeapscan.c unconditionally accesses it to - * perform prefetching. This probably needs to be rectified at a later - * point. - * * Optional callback, but either both scan_bitmap_next_block and * scan_bitmap_next_tuple need to exist, or neither. */ @@ -961,6 +950,9 @@ table_beginscan_bm(Relation rel, Snapshot snapshot, result = rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); result->shared_tbmiterator = NULL; result->tbmiterator = NULL; + result->pf_shared_tbmiterator = NULL; + result->pf_tbmiterator = NULL; + result->bm_parallel = NULL; return result; } @@ -1029,11 +1021,23 @@ table_endscan(TableScanDesc scan) scan->shared_tbmiterator = NULL; } + if (scan->pf_shared_tbmiterator) + { + tbm_end_shared_iterate(scan->pf_shared_tbmiterator); + scan->pf_shared_tbmiterator = NULL; + } + if (scan->tbmiterator) { tbm_end_iterate(scan->tbmiterator); scan->tbmiterator = NULL; } + + if (scan->pf_tbmiterator) + { + tbm_end_iterate(scan->pf_tbmiterator); + scan->pf_tbmiterator = NULL; + } } scan->rs_rd->rd_tableam->scan_end(scan); @@ -1054,11 +1058,23 @@ table_rescan(TableScanDesc scan, scan->shared_tbmiterator = NULL; } + if (scan->pf_shared_tbmiterator) + { + tbm_end_shared_iterate(scan->pf_shared_tbmiterator); + scan->pf_shared_tbmiterator = NULL; + } + if (scan->tbmiterator) { tbm_end_iterate(scan->tbmiterator); scan->tbmiterator = NULL; } + + if (scan->pf_tbmiterator) + { + tbm_end_iterate(scan->pf_tbmiterator); + scan->pf_tbmiterator = NULL; + } } scan->rs_rd->rd_tableam->scan_rescan(scan, key, false, false, false, false); diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 8688bc5ab0..7a3fdf9cd4 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1783,19 +1783,11 @@ typedef struct ParallelBitmapHeapState * * bitmapqualorig execution state for bitmapqualorig expressions * tbm bitmap obtained from child index scan(s) - * pvmbuffer buffer for visibility-map lookups of prefetched pages * exact_pages total number of exact pages retrieved * lossy_pages total number of lossy pages retrieved - * prefetch_iterator iterator for prefetching ahead of current page - * prefetch_pages # pages prefetch iterator is ahead of current - * prefetch_target current target prefetch distance - * prefetch_maximum maximum value for prefetch_target * initialized is node is ready to iterate - * shared_prefetch_iterator shared iterator for prefetching * pstate shared state for parallel bitmap scan * recheck do current page's tuples need recheck - * blockno used to validate pf and current block in sync - * pfblockno used to validate pf stays ahead of current block * ---------------- */ typedef struct BitmapHeapScanState @@ -1803,19 +1795,11 @@ typedef struct BitmapHeapScanState ScanState ss; /* its first field is NodeTag */ ExprState *bitmapqualorig; TIDBitmap *tbm; - Buffer pvmbuffer; long exact_pages; long lossy_pages; - TBMIterator *prefetch_iterator; - int prefetch_pages; - int prefetch_target; - int prefetch_maximum; bool initialized; - TBMSharedIterator *shared_prefetch_iterator; ParallelBitmapHeapState *pstate; bool recheck; - BlockNumber blockno; - BlockNumber pfblockno; } BitmapHeapScanState; /* ---------------- -- 2.40.1 --xqq4defy3uncu6k6 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v8-0014-Unify-parallel-and-serial-BitmapHeapScan-iterator.patch" ^ permalink raw reply [nested|flat] 10+ messages in thread
* [PATCH v11 14/17] Push BitmapHeapScan prefetch code into heapam.c @ 2024-03-22 13:42 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 10+ messages in thread From: Melanie Plageman @ 2024-03-22 13:42 UTC (permalink / raw) In preparation for transitioning to using the streaming read API for prefetching [1], move all of the BitmapHeapScanState members related to prefetching and the functions for accessing them into the HeapScanDescData and TableScanDescData. Members that still need to be accessed in BitmapHeapNext() could not be moved into heap AM-specific code. Specifically, parallel iterator setup requires several components which seem odd to pass to the table AM API. [1] https://www.postgresql.org/message-id/flat/CA%2BhUKGJkOiOCa%2Bmag4BF%2BzHo7qo%3Do9CFheB8%3Dg6uT5TUm2... --- src/backend/access/heap/heapam.c | 26 ++ src/backend/access/heap/heapam_handler.c | 262 +++++++++++++++++ src/backend/executor/nodeBitmapHeapscan.c | 341 ++-------------------- src/include/access/heapam.h | 17 ++ src/include/access/relscan.h | 8 + src/include/access/tableam.h | 26 +- src/include/nodes/execnodes.h | 14 - 7 files changed, 355 insertions(+), 339 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index ed3a3607b7..614d715fc7 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -948,8 +948,16 @@ heap_beginscan(Relation relation, Snapshot snapshot, scan->rs_base.rs_flags = flags; scan->rs_base.rs_parallel = parallel_scan; scan->rs_strategy = NULL; /* set in initscan */ + + scan->rs_base.blockno = InvalidBlockNumber; + scan->rs_vmbuffer = InvalidBuffer; scan->rs_empty_tuples_pending = 0; + scan->pvmbuffer = InvalidBuffer; + + scan->pfblockno = InvalidBlockNumber; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; /* * Disable page-at-a-time mode if it's not a MVCC-safe snapshot. @@ -1032,6 +1040,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, scan->rs_base.rs_flags &= ~SO_ALLOW_PAGEMODE; } + scan->rs_base.blockno = InvalidBlockNumber; + + scan->pfblockno = InvalidBlockNumber; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; + /* * unpin scan buffers */ @@ -1044,6 +1058,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, scan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(scan->pvmbuffer)) + { + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + } + /* * reinitialize scan descriptor */ @@ -1069,6 +1089,12 @@ heap_endscan(TableScanDesc sscan) scan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(scan->pvmbuffer)) + { + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + } + /* * decrement relation reference count and free scan descriptor storage */ diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index f7e4d1094d..68bbb6f88c 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -61,6 +61,9 @@ static bool SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, OffsetNumber tupoffset); static BlockNumber heapam_scan_get_blocks_done(HeapScanDesc hscan); +static inline void BitmapAdjustPrefetchIterator(HeapScanDesc scan); +static inline void BitmapAdjustPrefetchTarget(HeapScanDesc scan); +static inline void BitmapPrefetch(HeapScanDesc scan); static const TableAmRoutine heapam_methods; @@ -2170,6 +2173,73 @@ heapam_estimate_rel_size(Relation rel, int32 *attr_widths, * ------------------------------------------------------------------------ */ +/* + * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator + * + * We keep track of how far the prefetch iterator is ahead of the main + * iterator in prefetch_pages. For each block the main iterator returns, we + * decrement prefetch_pages. + */ +static inline void +BitmapAdjustPrefetchIterator(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + BitmapHeapIterator *prefetch_iterator = scan->rs_base.rs_pf_bhs_iterator; + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + TBMIterateResult tbmpre; + + if (pstate == NULL) + { + if (scan->prefetch_pages > 0) + { + /* The main iterator has closed the distance by one page */ + scan->prefetch_pages--; + } + else if (prefetch_iterator) + { + /* Do not let the prefetch iterator get behind the main one */ + bhs_iterate(prefetch_iterator, &tbmpre); + scan->pfblockno = tbmpre.blockno; + } + return; + } + + /* + * Adjusting the prefetch iterator before invoking + * table_scan_bitmap_next_block() keeps prefetch distance higher across + * the parallel workers. + */ + if (scan->rs_base.prefetch_maximum > 0) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages > 0) + { + pstate->prefetch_pages--; + SpinLockRelease(&pstate->mutex); + } + else + { + /* Release the mutex before iterating */ + SpinLockRelease(&pstate->mutex); + + /* + * In case of shared mode, we can not ensure that the current + * blockno of the main iterator and that of the prefetch iterator + * are same. It's possible that whatever blockno we are + * prefetching will be processed by another process. Therefore, + * we don't validate the blockno here as we do in non-parallel + * case. + */ + if (prefetch_iterator) + { + bhs_iterate(prefetch_iterator, &tbmpre); + scan->pfblockno = tbmpre.blockno; + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_block(TableScanDesc scan, bool *recheck, BlockNumber *blockno, @@ -2188,6 +2258,8 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, *blockno = InvalidBlockNumber; *recheck = true; + BitmapAdjustPrefetchIterator(hscan); + do { CHECK_FOR_INTERRUPTS(); @@ -2328,6 +2400,18 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, else (*exact_pages)++; + /* + * If serial, we can error out if the the prefetch block doesn't stay + * ahead of the current block. + */ + if (scan->bm_parallel == NULL && + scan->rs_pf_bhs_iterator && + hscan->pfblockno > hscan->rs_base.blockno) + elog(ERROR, "prefetch and main iterators are out of sync"); + + /* Adjust the prefetch target */ + BitmapAdjustPrefetchTarget(hscan); + /* * Return true to indicate that a valid block was found and the bitmap is * not exhausted. If there are no visible tuples on this page, @@ -2338,6 +2422,154 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, return true; } +/* + * BitmapAdjustPrefetchTarget - Adjust the prefetch target + * + * Increase prefetch target if it's not yet at the max. Note that + * we will increase it to zero after fetching the very first + * page/tuple, then to one after the second tuple is fetched, then + * it doubles as later pages are fetched. + */ +static inline void +BitmapAdjustPrefetchTarget(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + int prefetch_maximum = scan->rs_base.prefetch_maximum; + + if (pstate == NULL) + { + if (scan->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (scan->prefetch_target >= prefetch_maximum / 2) + scan->prefetch_target = prefetch_maximum; + else if (scan->prefetch_target > 0) + scan->prefetch_target *= 2; + else + scan->prefetch_target++; + return; + } + + /* Do an unlocked check first to save spinlock acquisitions. */ + if (pstate->prefetch_target < prefetch_maximum) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (pstate->prefetch_target >= prefetch_maximum / 2) + pstate->prefetch_target = prefetch_maximum; + else if (pstate->prefetch_target > 0) + pstate->prefetch_target *= 2; + else + pstate->prefetch_target++; + SpinLockRelease(&pstate->mutex); + } +#endif /* USE_PREFETCH */ +} + + +/* + * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target + */ +static inline void +BitmapPrefetch(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + BitmapHeapIterator *prefetch_iterator = scan->rs_base.rs_pf_bhs_iterator; + + if (pstate == NULL) + { + if (prefetch_iterator) + { + while (scan->prefetch_pages < scan->prefetch_target) + { + TBMIterateResult tbmpre; + bool skip_fetch; + + bhs_iterate(prefetch_iterator, &tbmpre); + + if (!BlockNumberIsValid(tbmpre.blockno)) + { + /* No more pages to prefetch */ + bhs_end_iterate(prefetch_iterator); + scan->rs_base.rs_pf_bhs_iterator = NULL; + break; + } + scan->prefetch_pages++; + scan->pfblockno = tbmpre.blockno; + + /* + * If we expect not to have to actually read this heap page, + * skip this prefetch call, but continue to run the prefetch + * logic normally. (Would it be better not to increment + * prefetch_pages?) + */ + skip_fetch = (!(scan->rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre.recheck && + VM_ALL_VISIBLE(scan->rs_base.rs_rd, + tbmpre.blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(scan->rs_base.rs_rd, MAIN_FORKNUM, tbmpre.blockno); + } + } + + return; + } + + if (pstate->prefetch_pages < pstate->prefetch_target) + { + if (prefetch_iterator) + { + while (1) + { + TBMIterateResult tbmpre; + bool do_prefetch = false; + bool skip_fetch; + + /* + * Recheck under the mutex. If some other process has already + * done enough prefetching then we need not to do anything. + */ + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages < pstate->prefetch_target) + { + pstate->prefetch_pages++; + do_prefetch = true; + } + SpinLockRelease(&pstate->mutex); + + if (!do_prefetch) + return; + + bhs_iterate(prefetch_iterator, &tbmpre); + if (!BlockNumberIsValid(tbmpre.blockno)) + { + /* No more pages to prefetch */ + bhs_end_iterate(prefetch_iterator); + scan->rs_base.rs_pf_bhs_iterator = NULL; + break; + } + + scan->pfblockno = tbmpre.blockno; + + /* As above, skip prefetch if we expect not to need page */ + skip_fetch = (!(scan->rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre.recheck && + VM_ALL_VISIBLE(scan->rs_base.rs_rd, + tbmpre.blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(scan->rs_base.rs_rd, MAIN_FORKNUM, tbmpre.blockno); + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_tuple(TableScanDesc scan, TupleTableSlot *slot) @@ -2363,6 +2595,36 @@ heapam_scan_bitmap_next_tuple(TableScanDesc scan, if (hscan->rs_cindex < 0 || hscan->rs_cindex >= hscan->rs_ntuples) return false; +#ifdef USE_PREFETCH + + /* + * Try to prefetch at least a few pages even before we get to the second + * page if we don't stop reading after the first tuple. + */ + if (!scan->bm_parallel) + { + if (hscan->prefetch_target < scan->prefetch_maximum) + hscan->prefetch_target++; + } + else if (scan->bm_parallel->prefetch_target < scan->prefetch_maximum) + { + /* take spinlock while updating shared state */ + SpinLockAcquire(&scan->bm_parallel->mutex); + if (scan->bm_parallel->prefetch_target < scan->prefetch_maximum) + scan->bm_parallel->prefetch_target++; + SpinLockRelease(&scan->bm_parallel->mutex); + } + + /* + * We issue prefetch requests *after* fetching the current page to try to + * avoid having prefetching interfere with the main I/O. Also, this should + * happen only when we have determined there is still something to do on + * the current page, else we may uselessly prefetch the same page we are + * just about to request for real. + */ + BitmapPrefetch(hscan); +#endif /* USE_PREFETCH */ + targoffset = hscan->rs_vistuples[hscan->rs_cindex]; page = BufferGetPage(hscan->rs_cbuf); lp = PageGetItemId(page, targoffset); diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index 78f79aafff..187b288e68 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -51,10 +51,6 @@ static TupleTableSlot *BitmapHeapNext(BitmapHeapScanState *node); static inline void BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate); -static inline void BitmapAdjustPrefetchIterator(BitmapHeapScanState *node); -static inline void BitmapAdjustPrefetchTarget(BitmapHeapScanState *node); -static inline void BitmapPrefetch(BitmapHeapScanState *node, - TableScanDesc scan); static bool BitmapShouldInitializeSharedState(ParallelBitmapHeapState *pstate); static BitmapHeapIterator *bhs_begin_iterate(TIDBitmap *tbm, dsa_pointer shared_area, @@ -122,7 +118,6 @@ BitmapHeapNext(BitmapHeapScanState *node) TableScanDesc scan; TIDBitmap *tbm; TupleTableSlot *slot; - ParallelBitmapHeapState *pstate = node->pstate; dsa_area *dsa = node->ss.ps.state->es_query_dsa; /* @@ -142,7 +137,7 @@ BitmapHeapNext(BitmapHeapScanState *node) * prefetching. node->prefetch_pages tracks exactly how many pages ahead * the prefetch iterator is. Also, node->prefetch_target tracks the * desired prefetch distance, which starts small and increases up to the - * node->prefetch_maximum. This is to avoid doing a lot of prefetching in + * scan->prefetch_maximum. This is to avoid doing a lot of prefetching in * a scan that stops after a few tuples because of a LIMIT. */ if (!node->initialized) @@ -154,7 +149,16 @@ BitmapHeapNext(BitmapHeapScanState *node) bool init_shared_state = node->pstate ? BitmapShouldInitializeSharedState(node->pstate) : false; - if (!pstate || init_shared_state) + /* + * Maximum number of prefetches for the tablespace if configured, + * otherwise the current value of the effective_io_concurrency GUC. + */ + int pf_maximum = 0; +#ifdef USE_PREFETCH + pf_maximum = get_tablespace_io_concurrency(node->ss.ss_currentRelation->rd_rel->reltablespace); +#endif + + if (!node->pstate || init_shared_state) { tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); @@ -169,23 +173,16 @@ BitmapHeapNext(BitmapHeapScanState *node) * dsa_pointer of the iterator state which will be used by * multiple processes to iterate jointly. */ - pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); + node->pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (pf_maximum > 0) { - pstate->prefetch_iterator = + node->pstate->prefetch_iterator = tbm_prepare_shared_iterate(tbm); - - /* - * We don't need the mutex here as we haven't yet woke up - * others. - */ - pstate->prefetch_pages = 0; - pstate->prefetch_target = -1; } #endif /* We have initialized the shared state so wake up others. */ - BitmapDoneInitializingSharedState(pstate); + BitmapDoneInitializingSharedState(node->pstate); } } @@ -216,19 +213,19 @@ BitmapHeapNext(BitmapHeapScanState *node) extra_flags); } + scan->prefetch_maximum = pf_maximum; + scan->bm_parallel = node->pstate; + scan->rs_bhs_iterator = bhs_begin_iterate(tbm, - pstate ? pstate->tbmiterator : InvalidDsaPointer, + scan->bm_parallel ? scan->bm_parallel->tbmiterator : InvalidDsaPointer, dsa); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (scan->prefetch_maximum > 0) { - node->pf_iterator = bhs_begin_iterate(tbm, - pstate ? pstate->prefetch_iterator : InvalidDsaPointer, - dsa); - /* Only used for serial BHS */ - node->prefetch_pages = 0; - node->prefetch_target = -1; + scan->rs_pf_bhs_iterator = bhs_begin_iterate(tbm, + scan->bm_parallel ? scan->bm_parallel->prefetch_iterator : InvalidDsaPointer, + dsa); } #endif /* USE_PREFETCH */ @@ -243,36 +240,6 @@ BitmapHeapNext(BitmapHeapScanState *node) { CHECK_FOR_INTERRUPTS(); -#ifdef USE_PREFETCH - - /* - * Try to prefetch at least a few pages even before we get to the - * second page if we don't stop reading after the first tuple. - */ - if (!pstate) - { - if (node->prefetch_target < node->prefetch_maximum) - node->prefetch_target++; - } - else if (pstate->prefetch_target < node->prefetch_maximum) - { - /* take spinlock while updating shared state */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target < node->prefetch_maximum) - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ - - /* - * We issue prefetch requests *after* fetching the current page to - * try to avoid having prefetching interfere with the main I/O. - * Also, this should happen only when we have determined there is - * still something to do on the current page, else we may - * uselessly prefetch the same page we are just about to request - * for real. - */ - BitmapPrefetch(node, scan); /* * If we are using lossy info, we have to recheck the qual @@ -296,23 +263,9 @@ BitmapHeapNext(BitmapHeapScanState *node) new_page: - BitmapAdjustPrefetchIterator(node); - - if (!table_scan_bitmap_next_block(scan, &node->recheck, &node->blockno, + if (!table_scan_bitmap_next_block(scan, &node->recheck, &scan->blockno, &node->lossy_pages, &node->exact_pages)) break; - - /* - * If serial, we can error out if the the prefetch block doesn't stay - * ahead of the current block. - */ - if (node->pstate == NULL && - node->pf_iterator && - node->pfblockno > node->blockno) - elog(ERROR, "prefetch and main iterators are out of sync"); - - /* Adjust the prefetch target */ - BitmapAdjustPrefetchTarget(node); } /* @@ -336,219 +289,6 @@ BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate) ConditionVariableBroadcast(&pstate->cv); } -/* - * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator - * - * We keep track of how far the prefetch iterator is ahead of the main - * iterator in prefetch_pages. For each block the main iterator returns, we - * decrement prefetch_pages. - */ -static inline void -BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - BitmapHeapIterator *prefetch_iterator = node->pf_iterator; - TBMIterateResult tbmpre; - - if (pstate == NULL) - { - if (node->prefetch_pages > 0) - { - /* The main iterator has closed the distance by one page */ - node->prefetch_pages--; - } - else if (prefetch_iterator) - { - /* Do not let the prefetch iterator get behind the main one */ - bhs_iterate(prefetch_iterator, &tbmpre); - node->pfblockno = tbmpre.blockno; - } - return; - } - - /* - * Adjusting the prefetch iterator before invoking - * table_scan_bitmap_next_block() keeps prefetch distance higher across - * the parallel workers. - */ - if (node->prefetch_maximum > 0) - { - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages > 0) - { - pstate->prefetch_pages--; - SpinLockRelease(&pstate->mutex); - } - else - { - /* Release the mutex before iterating */ - SpinLockRelease(&pstate->mutex); - - /* - * In case of shared mode, we can not ensure that the current - * blockno of the main iterator and that of the prefetch iterator - * are same. It's possible that whatever blockno we are - * prefetching will be processed by another process. Therefore, - * we don't validate the blockno here as we do in non-parallel - * case. - */ - if (prefetch_iterator) - { - bhs_iterate(prefetch_iterator, &tbmpre); - node->pfblockno = tbmpre.blockno; - } - } - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapAdjustPrefetchTarget - Adjust the prefetch target - * - * Increase prefetch target if it's not yet at the max. Note that - * we will increase it to zero after fetching the very first - * page/tuple, then to one after the second tuple is fetched, then - * it doubles as later pages are fetched. - */ -static inline void -BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - - if (pstate == NULL) - { - if (node->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (node->prefetch_target >= node->prefetch_maximum / 2) - node->prefetch_target = node->prefetch_maximum; - else if (node->prefetch_target > 0) - node->prefetch_target *= 2; - else - node->prefetch_target++; - return; - } - - /* Do an unlocked check first to save spinlock acquisitions. */ - if (pstate->prefetch_target < node->prefetch_maximum) - { - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (pstate->prefetch_target >= node->prefetch_maximum / 2) - pstate->prefetch_target = node->prefetch_maximum; - else if (pstate->prefetch_target > 0) - pstate->prefetch_target *= 2; - else - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target - */ -static inline void -BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - BitmapHeapIterator *prefetch_iterator = node->pf_iterator; - - if (pstate == NULL) - { - if (prefetch_iterator) - { - while (node->prefetch_pages < node->prefetch_target) - { - TBMIterateResult tbmpre; - bool skip_fetch; - - bhs_iterate(prefetch_iterator, &tbmpre); - - if (!BlockNumberIsValid(tbmpre.blockno)) - { - /* No more pages to prefetch */ - bhs_end_iterate(prefetch_iterator); - node->pf_iterator = NULL; - break; - } - node->prefetch_pages++; - node->pfblockno = tbmpre.blockno; - - /* - * If we expect not to have to actually read this heap page, - * skip this prefetch call, but continue to run the prefetch - * logic normally. (Would it be better not to increment - * prefetch_pages?) - */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre.recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre.blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre.blockno); - } - } - - return; - } - - if (pstate->prefetch_pages < pstate->prefetch_target) - { - if (prefetch_iterator) - { - while (1) - { - TBMIterateResult tbmpre; - bool do_prefetch = false; - bool skip_fetch; - - /* - * Recheck under the mutex. If some other process has already - * done enough prefetching then we need not to do anything. - */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages < pstate->prefetch_target) - { - pstate->prefetch_pages++; - do_prefetch = true; - } - SpinLockRelease(&pstate->mutex); - - if (!do_prefetch) - return; - - bhs_iterate(prefetch_iterator, &tbmpre); - if (!BlockNumberIsValid(tbmpre.blockno)) - { - /* No more pages to prefetch */ - bhs_end_iterate(prefetch_iterator); - node->pf_iterator = NULL; - break; - } - - node->pfblockno = tbmpre.blockno; - - /* As above, skip prefetch if we expect not to need page */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre.recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre.blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre.blockno); - } - } - } -#endif /* USE_PREFETCH */ -} - /* * BitmapHeapRecheck -- access method routine to recheck a tuple in EvalPlanQual */ @@ -594,22 +334,12 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) if (node->ss.ss_currentScanDesc) table_rescan(node->ss.ss_currentScanDesc, NULL); - /* release bitmaps and buffers if any */ - if (node->pf_iterator) - { - bhs_end_iterate(node->pf_iterator); - node->pf_iterator = NULL; - } + /* release bitmaps if any */ if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); node->tbm = NULL; node->initialized = false; - node->pvmbuffer = InvalidBuffer; node->recheck = true; - node->blockno = InvalidBlockNumber; - node->pfblockno = InvalidBlockNumber; ExecScanReScan(&node->ss); @@ -648,14 +378,10 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) table_endscan(scanDesc); /* - * release bitmaps and buffers if any + * release bitmaps if any */ - if (node->pf_iterator) - bhs_end_iterate(node->pf_iterator); if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); } /* ---------------------------------------------------------------- @@ -688,17 +414,11 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ps.ExecProcNode = ExecBitmapHeapScan; scanstate->tbm = NULL; - scanstate->pvmbuffer = InvalidBuffer; scanstate->exact_pages = 0; scanstate->lossy_pages = 0; - scanstate->pf_iterator = NULL; - scanstate->prefetch_pages = 0; - scanstate->prefetch_target = 0; scanstate->initialized = false; scanstate->pstate = NULL; scanstate->recheck = true; - scanstate->blockno = InvalidBlockNumber; - scanstate->pfblockno = InvalidBlockNumber; /* * Miscellaneous initialization @@ -738,13 +458,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->bitmapqualorig = ExecInitQual(node->bitmapqualorig, (PlanState *) scanstate); - /* - * Maximum number of prefetches for the tablespace if configured, - * otherwise the current value of the effective_io_concurrency GUC. - */ - scanstate->prefetch_maximum = - get_tablespace_io_concurrency(currentRelation->rd_rel->reltablespace); - scanstate->ss.ss_currentRelation = currentRelation; /* @@ -828,7 +541,7 @@ ExecBitmapHeapInitializeDSM(BitmapHeapScanState *node, /* Initialize the mutex */ SpinLockInit(&pstate->mutex); pstate->prefetch_pages = 0; - pstate->prefetch_target = 0; + pstate->prefetch_target = -1; pstate->state = BM_INITIAL; ConditionVariableInit(&pstate->cv); diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index c7a538221a..4726d31403 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -82,6 +82,23 @@ typedef struct HeapScanDescData Buffer rs_vmbuffer; int rs_empty_tuples_pending; + /* + * These fields only used for prefetching in bitmap table scans + */ + + /* buffer for visibility-map lookups of prefetched pages */ + Buffer pvmbuffer; + + /* + * These fields only used in serial BHS + */ + /* Current target for prefetch distance */ + int prefetch_target; + /* # pages prefetch iterator is ahead of current */ + int prefetch_pages; + /* used to validate prefetch block stays ahead of current block */ + BlockNumber pfblockno; + /* these fields only used in page-at-a-time mode and for bitmap scans */ int rs_cindex; /* current tuple's index in vistuples */ int rs_ntuples; /* number of visible tuples on page */ diff --git a/src/include/access/relscan.h b/src/include/access/relscan.h index fb22f305bf..7938b741d6 100644 --- a/src/include/access/relscan.h +++ b/src/include/access/relscan.h @@ -26,6 +26,7 @@ struct ParallelTableScanDescData; struct BitmapHeapIterator; +struct ParallelBitmapHeapState; /* * Generic descriptor for table scans. This is the base-class for table scans, @@ -45,6 +46,13 @@ typedef struct TableScanDescData /* Only used for Bitmap table scans */ struct BitmapHeapIterator *rs_bhs_iterator; + struct BitmapHeapIterator *rs_pf_bhs_iterator; + + /* maximum value for prefetch_target */ + int prefetch_maximum; + struct ParallelBitmapHeapState *bm_parallel; + /* used to validate BHS prefetch and current block stay in sync */ + BlockNumber blockno; /* * Information about type and behaviour of the scan, a bitmask of members diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index bb2b79717c..799ac013d4 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -811,17 +811,6 @@ typedef struct TableAmRoutine * lossy_pages is incremented if the block's representation in the bitmap * is lossy, otherwise, exact_pages is incremented. * - * XXX: Currently this may only be implemented if the AM uses md.c as its - * storage manager, and uses ItemPointer->ip_blkid in a manner that maps - * blockids directly to the underlying storage. nodeBitmapHeapscan.c - * performs prefetching directly using that interface. This probably - * needs to be rectified at a later point. - * - * XXX: Currently this may only be implemented if the AM uses the - * visibilitymap, as nodeBitmapHeapscan.c unconditionally accesses it to - * perform prefetching. This probably needs to be rectified at a later - * point. - * * Optional callback, but either both scan_bitmap_next_block and * scan_bitmap_next_tuple need to exist, or neither. */ @@ -971,6 +960,9 @@ table_beginscan_bm(Relation rel, Snapshot snapshot, result = rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); result->rs_bhs_iterator = NULL; + result->rs_pf_bhs_iterator = NULL; + result->prefetch_maximum = 0; + result->bm_parallel = NULL; return result; } @@ -1035,6 +1027,12 @@ table_endscan(TableScanDesc scan) { bhs_end_iterate(scan->rs_bhs_iterator); scan->rs_bhs_iterator = NULL; + + if (scan->rs_pf_bhs_iterator) + { + bhs_end_iterate(scan->rs_pf_bhs_iterator); + scan->rs_pf_bhs_iterator = NULL; + } } scan->rs_rd->rd_tableam->scan_end(scan); @@ -1051,6 +1049,12 @@ table_rescan(TableScanDesc scan, { bhs_end_iterate(scan->rs_bhs_iterator); scan->rs_bhs_iterator = NULL; + + if (scan->rs_pf_bhs_iterator) + { + bhs_end_iterate(scan->rs_pf_bhs_iterator); + scan->rs_pf_bhs_iterator = NULL; + } } scan->rs_rd->rd_tableam->scan_rescan(scan, key, false, false, false, false); diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 52cedd1b35..60916bf0d0 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1785,18 +1785,11 @@ struct BitmapHeapIterator; * * bitmapqualorig execution state for bitmapqualorig expressions * tbm bitmap obtained from child index scan(s) - * pvmbuffer buffer for visibility-map lookups of prefetched pages * exact_pages total number of exact pages retrieved * lossy_pages total number of lossy pages retrieved - * pf_iterator for prefetching ahead of current page - * prefetch_pages # pages prefetch iterator is ahead of current - * prefetch_target current target prefetch distance - * prefetch_maximum maximum value for prefetch_target * initialized is node is ready to iterate * pstate shared state for parallel bitmap scan * recheck do current page's tuples need recheck - * blockno used to validate pf and current block in sync - * pfblockno used to validate pf stays ahead of current block * ---------------- */ typedef struct BitmapHeapScanState @@ -1804,18 +1797,11 @@ typedef struct BitmapHeapScanState ScanState ss; /* its first field is NodeTag */ ExprState *bitmapqualorig; TIDBitmap *tbm; - Buffer pvmbuffer; long exact_pages; long lossy_pages; - int prefetch_pages; - int prefetch_target; - int prefetch_maximum; bool initialized; - struct BitmapHeapIterator *pf_iterator; ParallelBitmapHeapState *pstate; bool recheck; - BlockNumber blockno; - BlockNumber pfblockno; } BitmapHeapScanState; /* ---------------- -- 2.40.1 --owzzsiozz6hgpp7e Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v11-0015-Remove-table_scan_bitmap_next_block.patch" ^ permalink raw reply [nested|flat] 10+ messages in thread
* [PATCH v13 14/16] Push BitmapHeapScan prefetch code into heapam.c @ 2024-03-22 13:42 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 10+ messages in thread From: Melanie Plageman @ 2024-03-22 13:42 UTC (permalink / raw) In preparation for transitioning to using the streaming read API for prefetching [1], move all of the BitmapHeapScanState members related to prefetching and the functions for accessing them into the HeapScanDescData and TableScanDescData. Members that still need to be accessed in BitmapHeapNext() could not be moved into heap AM-specific code. Specifically, parallel iterator setup requires several components which seem odd to pass to the table AM API. [1] https://www.postgresql.org/message-id/flat/CA%2BhUKGJkOiOCa%2Bmag4BF%2BzHo7qo%3Do9CFheB8%3Dg6uT5TUm2... --- src/backend/access/heap/heapam.c | 26 ++ src/backend/access/heap/heapam_handler.c | 262 +++++++++++++++++ src/backend/executor/nodeBitmapHeapscan.c | 341 ++-------------------- src/include/access/heapam.h | 17 ++ src/include/access/relscan.h | 8 + src/include/access/tableam.h | 26 +- src/include/nodes/execnodes.h | 14 - 7 files changed, 355 insertions(+), 339 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 50b4bc1475..263c728543 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -948,8 +948,16 @@ heap_beginscan(Relation relation, Snapshot snapshot, scan->rs_base.rs_flags = flags; scan->rs_base.rs_parallel = parallel_scan; scan->rs_strategy = NULL; /* set in initscan */ + + scan->rs_base.blockno = InvalidBlockNumber; + scan->rs_vmbuffer = InvalidBuffer; scan->rs_empty_tuples_pending = 0; + scan->pvmbuffer = InvalidBuffer; + + scan->pfblockno = InvalidBlockNumber; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; /* * Disable page-at-a-time mode if it's not a MVCC-safe snapshot. @@ -1032,6 +1040,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, scan->rs_base.rs_flags &= ~SO_ALLOW_PAGEMODE; } + scan->rs_base.blockno = InvalidBlockNumber; + + scan->pfblockno = InvalidBlockNumber; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; + /* * unpin scan buffers */ @@ -1044,6 +1058,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, scan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(scan->pvmbuffer)) + { + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + } + /* * reinitialize scan descriptor */ @@ -1069,6 +1089,12 @@ heap_endscan(TableScanDesc sscan) scan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(scan->pvmbuffer)) + { + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + } + /* * decrement relation reference count and free scan descriptor storage */ diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index d9ceb4b848..42d5b749de 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -60,6 +60,9 @@ static bool SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, OffsetNumber tupoffset); static BlockNumber heapam_scan_get_blocks_done(HeapScanDesc hscan); +static inline void BitmapAdjustPrefetchIterator(HeapScanDesc scan); +static inline void BitmapAdjustPrefetchTarget(HeapScanDesc scan); +static inline void BitmapPrefetch(HeapScanDesc scan); static const TableAmRoutine heapam_methods; @@ -2186,6 +2189,73 @@ heapam_estimate_rel_size(Relation rel, int32 *attr_widths, * ------------------------------------------------------------------------ */ +/* + * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator + * + * We keep track of how far the prefetch iterator is ahead of the main + * iterator in prefetch_pages. For each block the main iterator returns, we + * decrement prefetch_pages. + */ +static inline void +BitmapAdjustPrefetchIterator(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + BitmapHeapIterator *prefetch_iterator = scan->rs_base.rs_pf_bhs_iterator; + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + TBMIterateResult tbmpre; + + if (pstate == NULL) + { + if (scan->prefetch_pages > 0) + { + /* The main iterator has closed the distance by one page */ + scan->prefetch_pages--; + } + else if (prefetch_iterator) + { + /* Do not let the prefetch iterator get behind the main one */ + bhs_iterate(prefetch_iterator, &tbmpre); + scan->pfblockno = tbmpre.blockno; + } + return; + } + + /* + * Adjusting the prefetch iterator before invoking + * table_scan_bitmap_next_block() keeps prefetch distance higher across + * the parallel workers. + */ + if (scan->rs_base.prefetch_maximum > 0) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages > 0) + { + pstate->prefetch_pages--; + SpinLockRelease(&pstate->mutex); + } + else + { + /* Release the mutex before iterating */ + SpinLockRelease(&pstate->mutex); + + /* + * In case of shared mode, we can not ensure that the current + * blockno of the main iterator and that of the prefetch iterator + * are same. It's possible that whatever blockno we are + * prefetching will be processed by another process. Therefore, + * we don't validate the blockno here as we do in non-parallel + * case. + */ + if (prefetch_iterator) + { + bhs_iterate(prefetch_iterator, &tbmpre); + scan->pfblockno = tbmpre.blockno; + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_block(TableScanDesc scan, bool *recheck, BlockNumber *blockno, @@ -2204,6 +2274,8 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, *blockno = InvalidBlockNumber; *recheck = true; + BitmapAdjustPrefetchIterator(hscan); + do { CHECK_FOR_INTERRUPTS(); @@ -2344,6 +2416,18 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, else (*exact_pages)++; + /* + * If serial, we can error out if the the prefetch block doesn't stay + * ahead of the current block. + */ + if (scan->bm_parallel == NULL && + scan->rs_pf_bhs_iterator && + hscan->pfblockno < hscan->rs_base.blockno) + elog(ERROR, "prefetch and main iterators are out of sync"); + + /* Adjust the prefetch target */ + BitmapAdjustPrefetchTarget(hscan); + /* * Return true to indicate that a valid block was found and the bitmap is * not exhausted. If there are no visible tuples on this page, @@ -2354,6 +2438,154 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, return true; } +/* + * BitmapAdjustPrefetchTarget - Adjust the prefetch target + * + * Increase prefetch target if it's not yet at the max. Note that + * we will increase it to zero after fetching the very first + * page/tuple, then to one after the second tuple is fetched, then + * it doubles as later pages are fetched. + */ +static inline void +BitmapAdjustPrefetchTarget(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + int prefetch_maximum = scan->rs_base.prefetch_maximum; + + if (pstate == NULL) + { + if (scan->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (scan->prefetch_target >= prefetch_maximum / 2) + scan->prefetch_target = prefetch_maximum; + else if (scan->prefetch_target > 0) + scan->prefetch_target *= 2; + else + scan->prefetch_target++; + return; + } + + /* Do an unlocked check first to save spinlock acquisitions. */ + if (pstate->prefetch_target < prefetch_maximum) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (pstate->prefetch_target >= prefetch_maximum / 2) + pstate->prefetch_target = prefetch_maximum; + else if (pstate->prefetch_target > 0) + pstate->prefetch_target *= 2; + else + pstate->prefetch_target++; + SpinLockRelease(&pstate->mutex); + } +#endif /* USE_PREFETCH */ +} + + +/* + * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target + */ +static inline void +BitmapPrefetch(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + BitmapHeapIterator *prefetch_iterator = scan->rs_base.rs_pf_bhs_iterator; + + if (pstate == NULL) + { + if (prefetch_iterator) + { + while (scan->prefetch_pages < scan->prefetch_target) + { + TBMIterateResult tbmpre; + bool skip_fetch; + + bhs_iterate(prefetch_iterator, &tbmpre); + + if (!BlockNumberIsValid(tbmpre.blockno)) + { + /* No more pages to prefetch */ + bhs_end_iterate(prefetch_iterator); + scan->rs_base.rs_pf_bhs_iterator = NULL; + break; + } + scan->prefetch_pages++; + scan->pfblockno = tbmpre.blockno; + + /* + * If we expect not to have to actually read this heap page, + * skip this prefetch call, but continue to run the prefetch + * logic normally. (Would it be better not to increment + * prefetch_pages?) + */ + skip_fetch = (!(scan->rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre.recheck && + VM_ALL_VISIBLE(scan->rs_base.rs_rd, + tbmpre.blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(scan->rs_base.rs_rd, MAIN_FORKNUM, tbmpre.blockno); + } + } + + return; + } + + if (pstate->prefetch_pages < pstate->prefetch_target) + { + if (prefetch_iterator) + { + while (1) + { + TBMIterateResult tbmpre; + bool do_prefetch = false; + bool skip_fetch; + + /* + * Recheck under the mutex. If some other process has already + * done enough prefetching then we need not to do anything. + */ + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages < pstate->prefetch_target) + { + pstate->prefetch_pages++; + do_prefetch = true; + } + SpinLockRelease(&pstate->mutex); + + if (!do_prefetch) + return; + + bhs_iterate(prefetch_iterator, &tbmpre); + if (!BlockNumberIsValid(tbmpre.blockno)) + { + /* No more pages to prefetch */ + bhs_end_iterate(prefetch_iterator); + scan->rs_base.rs_pf_bhs_iterator = NULL; + break; + } + + scan->pfblockno = tbmpre.blockno; + + /* As above, skip prefetch if we expect not to need page */ + skip_fetch = (!(scan->rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre.recheck && + VM_ALL_VISIBLE(scan->rs_base.rs_rd, + tbmpre.blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(scan->rs_base.rs_rd, MAIN_FORKNUM, tbmpre.blockno); + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_tuple(TableScanDesc scan, TupleTableSlot *slot) @@ -2379,6 +2611,36 @@ heapam_scan_bitmap_next_tuple(TableScanDesc scan, if (hscan->rs_cindex < 0 || hscan->rs_cindex >= hscan->rs_ntuples) return false; +#ifdef USE_PREFETCH + + /* + * Try to prefetch at least a few pages even before we get to the second + * page if we don't stop reading after the first tuple. + */ + if (!scan->bm_parallel) + { + if (hscan->prefetch_target < scan->prefetch_maximum) + hscan->prefetch_target++; + } + else if (scan->bm_parallel->prefetch_target < scan->prefetch_maximum) + { + /* take spinlock while updating shared state */ + SpinLockAcquire(&scan->bm_parallel->mutex); + if (scan->bm_parallel->prefetch_target < scan->prefetch_maximum) + scan->bm_parallel->prefetch_target++; + SpinLockRelease(&scan->bm_parallel->mutex); + } + + /* + * We issue prefetch requests *after* fetching the current page to try to + * avoid having prefetching interfere with the main I/O. Also, this should + * happen only when we have determined there is still something to do on + * the current page, else we may uselessly prefetch the same page we are + * just about to request for real. + */ + BitmapPrefetch(hscan); +#endif /* USE_PREFETCH */ + targoffset = hscan->rs_vistuples[hscan->rs_cindex]; page = BufferGetPage(hscan->rs_cbuf); lp = PageGetItemId(page, targoffset); diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index d61965a276..187b288e68 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -51,10 +51,6 @@ static TupleTableSlot *BitmapHeapNext(BitmapHeapScanState *node); static inline void BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate); -static inline void BitmapAdjustPrefetchIterator(BitmapHeapScanState *node); -static inline void BitmapAdjustPrefetchTarget(BitmapHeapScanState *node); -static inline void BitmapPrefetch(BitmapHeapScanState *node, - TableScanDesc scan); static bool BitmapShouldInitializeSharedState(ParallelBitmapHeapState *pstate); static BitmapHeapIterator *bhs_begin_iterate(TIDBitmap *tbm, dsa_pointer shared_area, @@ -122,7 +118,6 @@ BitmapHeapNext(BitmapHeapScanState *node) TableScanDesc scan; TIDBitmap *tbm; TupleTableSlot *slot; - ParallelBitmapHeapState *pstate = node->pstate; dsa_area *dsa = node->ss.ps.state->es_query_dsa; /* @@ -142,7 +137,7 @@ BitmapHeapNext(BitmapHeapScanState *node) * prefetching. node->prefetch_pages tracks exactly how many pages ahead * the prefetch iterator is. Also, node->prefetch_target tracks the * desired prefetch distance, which starts small and increases up to the - * node->prefetch_maximum. This is to avoid doing a lot of prefetching in + * scan->prefetch_maximum. This is to avoid doing a lot of prefetching in * a scan that stops after a few tuples because of a LIMIT. */ if (!node->initialized) @@ -154,7 +149,16 @@ BitmapHeapNext(BitmapHeapScanState *node) bool init_shared_state = node->pstate ? BitmapShouldInitializeSharedState(node->pstate) : false; - if (!pstate || init_shared_state) + /* + * Maximum number of prefetches for the tablespace if configured, + * otherwise the current value of the effective_io_concurrency GUC. + */ + int pf_maximum = 0; +#ifdef USE_PREFETCH + pf_maximum = get_tablespace_io_concurrency(node->ss.ss_currentRelation->rd_rel->reltablespace); +#endif + + if (!node->pstate || init_shared_state) { tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); @@ -169,23 +173,16 @@ BitmapHeapNext(BitmapHeapScanState *node) * dsa_pointer of the iterator state which will be used by * multiple processes to iterate jointly. */ - pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); + node->pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (pf_maximum > 0) { - pstate->prefetch_iterator = + node->pstate->prefetch_iterator = tbm_prepare_shared_iterate(tbm); - - /* - * We don't need the mutex here as we haven't yet woke up - * others. - */ - pstate->prefetch_pages = 0; - pstate->prefetch_target = -1; } #endif /* We have initialized the shared state so wake up others. */ - BitmapDoneInitializingSharedState(pstate); + BitmapDoneInitializingSharedState(node->pstate); } } @@ -216,19 +213,19 @@ BitmapHeapNext(BitmapHeapScanState *node) extra_flags); } + scan->prefetch_maximum = pf_maximum; + scan->bm_parallel = node->pstate; + scan->rs_bhs_iterator = bhs_begin_iterate(tbm, - pstate ? pstate->tbmiterator : InvalidDsaPointer, + scan->bm_parallel ? scan->bm_parallel->tbmiterator : InvalidDsaPointer, dsa); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (scan->prefetch_maximum > 0) { - node->pf_iterator = bhs_begin_iterate(tbm, - pstate ? pstate->prefetch_iterator : InvalidDsaPointer, - dsa); - /* Only used for serial BHS */ - node->prefetch_pages = 0; - node->prefetch_target = -1; + scan->rs_pf_bhs_iterator = bhs_begin_iterate(tbm, + scan->bm_parallel ? scan->bm_parallel->prefetch_iterator : InvalidDsaPointer, + dsa); } #endif /* USE_PREFETCH */ @@ -243,36 +240,6 @@ BitmapHeapNext(BitmapHeapScanState *node) { CHECK_FOR_INTERRUPTS(); -#ifdef USE_PREFETCH - - /* - * Try to prefetch at least a few pages even before we get to the - * second page if we don't stop reading after the first tuple. - */ - if (!pstate) - { - if (node->prefetch_target < node->prefetch_maximum) - node->prefetch_target++; - } - else if (pstate->prefetch_target < node->prefetch_maximum) - { - /* take spinlock while updating shared state */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target < node->prefetch_maximum) - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ - - /* - * We issue prefetch requests *after* fetching the current page to - * try to avoid having prefetching interfere with the main I/O. - * Also, this should happen only when we have determined there is - * still something to do on the current page, else we may - * uselessly prefetch the same page we are just about to request - * for real. - */ - BitmapPrefetch(node, scan); /* * If we are using lossy info, we have to recheck the qual @@ -296,23 +263,9 @@ BitmapHeapNext(BitmapHeapScanState *node) new_page: - BitmapAdjustPrefetchIterator(node); - - if (!table_scan_bitmap_next_block(scan, &node->recheck, &node->blockno, + if (!table_scan_bitmap_next_block(scan, &node->recheck, &scan->blockno, &node->lossy_pages, &node->exact_pages)) break; - - /* - * If serial, we can error out if the the prefetch block doesn't stay - * ahead of the current block. - */ - if (node->pstate == NULL && - node->pf_iterator && - node->pfblockno < node->blockno) - elog(ERROR, "prefetch and main iterators are out of sync"); - - /* Adjust the prefetch target */ - BitmapAdjustPrefetchTarget(node); } /* @@ -336,219 +289,6 @@ BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate) ConditionVariableBroadcast(&pstate->cv); } -/* - * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator - * - * We keep track of how far the prefetch iterator is ahead of the main - * iterator in prefetch_pages. For each block the main iterator returns, we - * decrement prefetch_pages. - */ -static inline void -BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - BitmapHeapIterator *prefetch_iterator = node->pf_iterator; - TBMIterateResult tbmpre; - - if (pstate == NULL) - { - if (node->prefetch_pages > 0) - { - /* The main iterator has closed the distance by one page */ - node->prefetch_pages--; - } - else if (prefetch_iterator) - { - /* Do not let the prefetch iterator get behind the main one */ - bhs_iterate(prefetch_iterator, &tbmpre); - node->pfblockno = tbmpre.blockno; - } - return; - } - - /* - * Adjusting the prefetch iterator before invoking - * table_scan_bitmap_next_block() keeps prefetch distance higher across - * the parallel workers. - */ - if (node->prefetch_maximum > 0) - { - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages > 0) - { - pstate->prefetch_pages--; - SpinLockRelease(&pstate->mutex); - } - else - { - /* Release the mutex before iterating */ - SpinLockRelease(&pstate->mutex); - - /* - * In case of shared mode, we can not ensure that the current - * blockno of the main iterator and that of the prefetch iterator - * are same. It's possible that whatever blockno we are - * prefetching will be processed by another process. Therefore, - * we don't validate the blockno here as we do in non-parallel - * case. - */ - if (prefetch_iterator) - { - bhs_iterate(prefetch_iterator, &tbmpre); - node->pfblockno = tbmpre.blockno; - } - } - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapAdjustPrefetchTarget - Adjust the prefetch target - * - * Increase prefetch target if it's not yet at the max. Note that - * we will increase it to zero after fetching the very first - * page/tuple, then to one after the second tuple is fetched, then - * it doubles as later pages are fetched. - */ -static inline void -BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - - if (pstate == NULL) - { - if (node->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (node->prefetch_target >= node->prefetch_maximum / 2) - node->prefetch_target = node->prefetch_maximum; - else if (node->prefetch_target > 0) - node->prefetch_target *= 2; - else - node->prefetch_target++; - return; - } - - /* Do an unlocked check first to save spinlock acquisitions. */ - if (pstate->prefetch_target < node->prefetch_maximum) - { - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (pstate->prefetch_target >= node->prefetch_maximum / 2) - pstate->prefetch_target = node->prefetch_maximum; - else if (pstate->prefetch_target > 0) - pstate->prefetch_target *= 2; - else - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target - */ -static inline void -BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - BitmapHeapIterator *prefetch_iterator = node->pf_iterator; - - if (pstate == NULL) - { - if (prefetch_iterator) - { - while (node->prefetch_pages < node->prefetch_target) - { - TBMIterateResult tbmpre; - bool skip_fetch; - - bhs_iterate(prefetch_iterator, &tbmpre); - - if (!BlockNumberIsValid(tbmpre.blockno)) - { - /* No more pages to prefetch */ - bhs_end_iterate(prefetch_iterator); - node->pf_iterator = NULL; - break; - } - node->prefetch_pages++; - node->pfblockno = tbmpre.blockno; - - /* - * If we expect not to have to actually read this heap page, - * skip this prefetch call, but continue to run the prefetch - * logic normally. (Would it be better not to increment - * prefetch_pages?) - */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre.recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre.blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre.blockno); - } - } - - return; - } - - if (pstate->prefetch_pages < pstate->prefetch_target) - { - if (prefetch_iterator) - { - while (1) - { - TBMIterateResult tbmpre; - bool do_prefetch = false; - bool skip_fetch; - - /* - * Recheck under the mutex. If some other process has already - * done enough prefetching then we need not to do anything. - */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages < pstate->prefetch_target) - { - pstate->prefetch_pages++; - do_prefetch = true; - } - SpinLockRelease(&pstate->mutex); - - if (!do_prefetch) - return; - - bhs_iterate(prefetch_iterator, &tbmpre); - if (!BlockNumberIsValid(tbmpre.blockno)) - { - /* No more pages to prefetch */ - bhs_end_iterate(prefetch_iterator); - node->pf_iterator = NULL; - break; - } - - node->pfblockno = tbmpre.blockno; - - /* As above, skip prefetch if we expect not to need page */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre.recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre.blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre.blockno); - } - } - } -#endif /* USE_PREFETCH */ -} - /* * BitmapHeapRecheck -- access method routine to recheck a tuple in EvalPlanQual */ @@ -594,22 +334,12 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) if (node->ss.ss_currentScanDesc) table_rescan(node->ss.ss_currentScanDesc, NULL); - /* release bitmaps and buffers if any */ - if (node->pf_iterator) - { - bhs_end_iterate(node->pf_iterator); - node->pf_iterator = NULL; - } + /* release bitmaps if any */ if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); node->tbm = NULL; node->initialized = false; - node->pvmbuffer = InvalidBuffer; node->recheck = true; - node->blockno = InvalidBlockNumber; - node->pfblockno = InvalidBlockNumber; ExecScanReScan(&node->ss); @@ -648,14 +378,10 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) table_endscan(scanDesc); /* - * release bitmaps and buffers if any + * release bitmaps if any */ - if (node->pf_iterator) - bhs_end_iterate(node->pf_iterator); if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); } /* ---------------------------------------------------------------- @@ -688,17 +414,11 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ps.ExecProcNode = ExecBitmapHeapScan; scanstate->tbm = NULL; - scanstate->pvmbuffer = InvalidBuffer; scanstate->exact_pages = 0; scanstate->lossy_pages = 0; - scanstate->pf_iterator = NULL; - scanstate->prefetch_pages = 0; - scanstate->prefetch_target = 0; scanstate->initialized = false; scanstate->pstate = NULL; scanstate->recheck = true; - scanstate->blockno = InvalidBlockNumber; - scanstate->pfblockno = InvalidBlockNumber; /* * Miscellaneous initialization @@ -738,13 +458,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->bitmapqualorig = ExecInitQual(node->bitmapqualorig, (PlanState *) scanstate); - /* - * Maximum number of prefetches for the tablespace if configured, - * otherwise the current value of the effective_io_concurrency GUC. - */ - scanstate->prefetch_maximum = - get_tablespace_io_concurrency(currentRelation->rd_rel->reltablespace); - scanstate->ss.ss_currentRelation = currentRelation; /* @@ -828,7 +541,7 @@ ExecBitmapHeapInitializeDSM(BitmapHeapScanState *node, /* Initialize the mutex */ SpinLockInit(&pstate->mutex); pstate->prefetch_pages = 0; - pstate->prefetch_target = 0; + pstate->prefetch_target = -1; pstate->state = BM_INITIAL; ConditionVariableInit(&pstate->cv); diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index a527e1b99f..bf212f577f 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -86,6 +86,23 @@ typedef struct HeapScanDescData Buffer rs_vmbuffer; int rs_empty_tuples_pending; + /* + * These fields only used for prefetching in bitmap table scans + */ + + /* buffer for visibility-map lookups of prefetched pages */ + Buffer pvmbuffer; + + /* + * These fields only used in serial BHS + */ + /* Current target for prefetch distance */ + int prefetch_target; + /* # pages prefetch iterator is ahead of current */ + int prefetch_pages; + /* used to validate prefetch block stays ahead of current block */ + BlockNumber pfblockno; + /* these fields only used in page-at-a-time mode and for bitmap scans */ int rs_cindex; /* current tuple's index in vistuples */ int rs_ntuples; /* number of visible tuples on page */ diff --git a/src/include/access/relscan.h b/src/include/access/relscan.h index fb22f305bf..7938b741d6 100644 --- a/src/include/access/relscan.h +++ b/src/include/access/relscan.h @@ -26,6 +26,7 @@ struct ParallelTableScanDescData; struct BitmapHeapIterator; +struct ParallelBitmapHeapState; /* * Generic descriptor for table scans. This is the base-class for table scans, @@ -45,6 +46,13 @@ typedef struct TableScanDescData /* Only used for Bitmap table scans */ struct BitmapHeapIterator *rs_bhs_iterator; + struct BitmapHeapIterator *rs_pf_bhs_iterator; + + /* maximum value for prefetch_target */ + int prefetch_maximum; + struct ParallelBitmapHeapState *bm_parallel; + /* used to validate BHS prefetch and current block stay in sync */ + BlockNumber blockno; /* * Information about type and behaviour of the scan, a bitmask of members diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 70e538b76b..a2b229fb87 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -785,17 +785,6 @@ typedef struct TableAmRoutine * lossy_pages is incremented if the block's representation in the bitmap * is lossy, otherwise, exact_pages is incremented. * - * XXX: Currently this may only be implemented if the AM uses md.c as its - * storage manager, and uses ItemPointer->ip_blkid in a manner that maps - * blockids directly to the underlying storage. nodeBitmapHeapscan.c - * performs prefetching directly using that interface. This probably - * needs to be rectified at a later point. - * - * XXX: Currently this may only be implemented if the AM uses the - * visibilitymap, as nodeBitmapHeapscan.c unconditionally accesses it to - * perform prefetching. This probably needs to be rectified at a later - * point. - * * Optional callback, but either both scan_bitmap_next_block and * scan_bitmap_next_tuple need to exist, or neither. */ @@ -945,6 +934,9 @@ table_beginscan_bm(Relation rel, Snapshot snapshot, result = rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); result->rs_bhs_iterator = NULL; + result->rs_pf_bhs_iterator = NULL; + result->prefetch_maximum = 0; + result->bm_parallel = NULL; return result; } @@ -996,6 +988,12 @@ table_endscan(TableScanDesc scan) { bhs_end_iterate(scan->rs_bhs_iterator); scan->rs_bhs_iterator = NULL; + + if (scan->rs_pf_bhs_iterator) + { + bhs_end_iterate(scan->rs_pf_bhs_iterator); + scan->rs_pf_bhs_iterator = NULL; + } } scan->rs_rd->rd_tableam->scan_end(scan); @@ -1012,6 +1010,12 @@ table_rescan(TableScanDesc scan, { bhs_end_iterate(scan->rs_bhs_iterator); scan->rs_bhs_iterator = NULL; + + if (scan->rs_pf_bhs_iterator) + { + bhs_end_iterate(scan->rs_pf_bhs_iterator); + scan->rs_pf_bhs_iterator = NULL; + } } scan->rs_rd->rd_tableam->scan_rescan(scan, key, false, false, false, false); diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index cf8b4995f0..592215d5ee 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1794,18 +1794,11 @@ struct BitmapHeapIterator; * * bitmapqualorig execution state for bitmapqualorig expressions * tbm bitmap obtained from child index scan(s) - * pvmbuffer buffer for visibility-map lookups of prefetched pages * exact_pages total number of exact pages retrieved * lossy_pages total number of lossy pages retrieved - * pf_iterator for prefetching ahead of current page - * prefetch_pages # pages prefetch iterator is ahead of current - * prefetch_target current target prefetch distance - * prefetch_maximum maximum value for prefetch_target * initialized is node is ready to iterate * pstate shared state for parallel bitmap scan * recheck do current page's tuples need recheck - * blockno used to validate pf and current block in sync - * pfblockno used to validate pf stays ahead of current block * ---------------- */ typedef struct BitmapHeapScanState @@ -1813,18 +1806,11 @@ typedef struct BitmapHeapScanState ScanState ss; /* its first field is NodeTag */ ExprState *bitmapqualorig; TIDBitmap *tbm; - Buffer pvmbuffer; long exact_pages; long lossy_pages; - int prefetch_pages; - int prefetch_target; - int prefetch_maximum; bool initialized; - struct BitmapHeapIterator *pf_iterator; ParallelBitmapHeapState *pstate; bool recheck; - BlockNumber blockno; - BlockNumber pfblockno; } BitmapHeapScanState; /* ---------------- -- 2.40.1 --cuuqjeyokkhgd736 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0015-Remove-table_scan_bitmap_next_block.patch" ^ permalink raw reply [nested|flat] 10+ messages in thread
* [PATCH v15 11/13] Push BitmapHeapScan prefetch code into heapam.c @ 2024-03-22 13:42 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 10+ messages in thread From: Melanie Plageman @ 2024-03-22 13:42 UTC (permalink / raw) 7566751f0ce9 eliminated a table AM layering violation for the current block in a BitmapHeapScan but left the violation in BitmapHeapScan prefetching code. To resolve this, move the BitmapHeapScan prefetching logic entirely into heap AM code. There is a fair amount of setup that is specific to BitmapHeapScan and once the state associated with that lives in the scan descriptor, it made more sense to make dedicated begin_scan()/rescan()/endscan() routines for BitmapHeapScan. Author: Melanie Plageman Discussion: https://postgr.es/m/CAAKRu_ZwCwWFeL_H3ia26bP2e7HiKLWt0ZmGXPVwPO6uXq0vaA%40mail.gmail.com --- src/backend/access/heap/heapam.c | 129 ++++++- src/backend/access/heap/heapam_handler.c | 306 ++++++++++++++-- src/backend/executor/nodeBitmapHeapscan.c | 410 +++------------------- src/include/access/heapam.h | 62 +++- src/include/access/relscan.h | 3 - src/include/access/tableam.h | 74 ++-- src/include/executor/nodeBitmapHeapscan.h | 6 + src/include/nodes/execnodes.h | 21 -- src/tools/pgindent/typedefs.list | 1 + 9 files changed, 553 insertions(+), 459 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 10f2faaa60..a21ec92d71 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -938,6 +938,48 @@ continue_page: * ---------------------------------------------------------------- */ +TableScanDesc +heap_beginscan_bm(Relation relation, Snapshot snapshot, uint32 flags) +{ + BitmapHeapScanDesc scan; + + /* + * increment relation ref count while scanning relation + * + * This is just to make really sure the relcache entry won't go away while + * the scan has a pointer to it. Caller should be holding the rel open + * anyway, so this is redundant in all normal scenarios... + */ + RelationIncrementReferenceCount(relation); + scan = (BitmapHeapScanDesc) palloc(sizeof(BitmapHeapScanDescData)); + + scan->heap_common.rs_base.rs_rd = relation; + scan->heap_common.rs_base.rs_snapshot = snapshot; + scan->heap_common.rs_base.rs_nkeys = 0; + scan->heap_common.rs_base.rs_flags = flags; + scan->heap_common.rs_base.rs_parallel = NULL; + scan->heap_common.rs_strategy = NULL; + + Assert(snapshot && IsMVCCSnapshot(snapshot)); + + /* we only need to set this up once */ + scan->heap_common.rs_ctup.t_tableOid = RelationGetRelid(relation); + + scan->heap_common.rs_parallelworkerdata = NULL; + scan->heap_common.rs_base.rs_key = NULL; + + initscan(&scan->heap_common, NULL, false); + + scan->iterator.serial = NULL; + scan->iterator.parallel = NULL; + scan->vmbuffer = InvalidBuffer; + scan->pf_iterator.serial = NULL; + scan->pf_iterator.parallel = NULL; + scan->pvmbuffer = InvalidBuffer; + + return (TableScanDesc) scan; +} + TableScanDesc heap_beginscan(Relation relation, Snapshot snapshot, @@ -967,8 +1009,6 @@ heap_beginscan(Relation relation, Snapshot snapshot, scan->rs_base.rs_flags = flags; scan->rs_base.rs_parallel = parallel_scan; scan->rs_strategy = NULL; /* set in initscan */ - scan->rs_vmbuffer = InvalidBuffer; - scan->rs_empty_tuples_pending = 0; /* * Disable page-at-a-time mode if it's not a MVCC-safe snapshot. @@ -1026,6 +1066,35 @@ heap_beginscan(Relation relation, Snapshot snapshot, return (TableScanDesc) scan; } +/* + * Cleanup BitmapHeapScan table state + */ +void +heap_endscan_bm(TableScanDesc sscan) +{ + BitmapHeapScanDesc scan = (BitmapHeapScanDesc) sscan; + + if (BufferIsValid(scan->heap_common.rs_cbuf)) + ReleaseBuffer(scan->heap_common.rs_cbuf); + + if (BufferIsValid(scan->vmbuffer)) + ReleaseBuffer(scan->vmbuffer); + + if (BufferIsValid(scan->pvmbuffer)) + ReleaseBuffer(scan->pvmbuffer); + + bhs_end_iterate(&scan->pf_iterator); + bhs_end_iterate(&scan->iterator); + + /* + * decrement relation reference count and free scan descriptor storage + */ + RelationDecrementReferenceCount(scan->heap_common.rs_base.rs_rd); + + pfree(scan); +} + + void heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, bool allow_strat, bool allow_sync, bool allow_pagemode) @@ -1057,12 +1126,6 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, if (BufferIsValid(scan->rs_cbuf)) ReleaseBuffer(scan->rs_cbuf); - if (BufferIsValid(scan->rs_vmbuffer)) - { - ReleaseBuffer(scan->rs_vmbuffer); - scan->rs_vmbuffer = InvalidBuffer; - } - /* * reinitialize scan descriptor */ @@ -1082,12 +1145,6 @@ heap_endscan(TableScanDesc sscan) if (BufferIsValid(scan->rs_cbuf)) ReleaseBuffer(scan->rs_cbuf); - if (BufferIsValid(scan->rs_vmbuffer)) - { - ReleaseBuffer(scan->rs_vmbuffer); - scan->rs_vmbuffer = InvalidBuffer; - } - /* * decrement relation reference count and free scan descriptor storage */ @@ -1334,6 +1391,50 @@ heap_getnextslot_tidrange(TableScanDesc sscan, ScanDirection direction, return true; } +void +heap_rescan_bm(TableScanDesc sscan, TIDBitmap *tbm, + ParallelBitmapHeapState *pstate, dsa_area *dsa, int pf_maximum) +{ + BitmapHeapScanDesc scan = (BitmapHeapScanDesc) sscan; + + if (BufferIsValid(scan->vmbuffer)) + ReleaseBuffer(scan->vmbuffer); + scan->vmbuffer = InvalidBuffer; + + if (BufferIsValid(scan->pvmbuffer)) + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + + bhs_end_iterate(&scan->pf_iterator); + bhs_end_iterate(&scan->iterator); + + scan->prefetch_maximum = 0; + scan->empty_tuples_pending = 0; + scan->pstate = NULL; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; + scan->pfblockno = InvalidBlockNumber; + + bhs_begin_iterate(&scan->iterator, + tbm, dsa, + pstate ? + pstate->tbmiterator : + InvalidDsaPointer); +#ifdef USE_PREFETCH + if (pf_maximum > 0) + { + bhs_begin_iterate(&scan->pf_iterator, tbm, dsa, + pstate ? + pstate->prefetch_iterator : + InvalidDsaPointer); + } +#endif /* USE_PREFETCH */ + + scan->pstate = pstate; + scan->prefetch_maximum = pf_maximum; +} + + /* * heap_fetch - retrieve tuple with given tid * diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 5e57d19d91..c513cbd2da 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -60,6 +60,9 @@ static bool SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, OffsetNumber tupoffset); static BlockNumber heapam_scan_get_blocks_done(HeapScanDesc hscan); +static inline void BitmapAdjustPrefetchIterator(BitmapHeapScanDesc scan); +static inline void BitmapAdjustPrefetchTarget(BitmapHeapScanDesc scan); +static inline void BitmapPrefetch(BitmapHeapScanDesc scan); static const TableAmRoutine heapam_methods; @@ -2186,12 +2189,80 @@ heapam_estimate_rel_size(Relation rel, int32 *attr_widths, * ------------------------------------------------------------------------ */ +/* + * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator + * + * We keep track of how far the prefetch iterator is ahead of the main + * iterator in prefetch_pages. For each block the main iterator returns, we + * decrement prefetch_pages. + */ +static inline void +BitmapAdjustPrefetchIterator(BitmapHeapScanDesc scan) +{ +#ifdef USE_PREFETCH + BitmapHeapIterator *prefetch_iterator = &scan->pf_iterator; + ParallelBitmapHeapState *pstate = scan->pstate; + TBMIterateResult *tbmpre; + + if (pstate == NULL) + { + if (scan->prefetch_pages > 0) + { + /* The main iterator has closed the distance by one page */ + scan->prefetch_pages--; + } + else if (!prefetch_iterator->exhausted) + { + /* Do not let the prefetch iterator get behind the main one */ + tbmpre = bhs_iterate(prefetch_iterator); + scan->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; + } + return; + } + + /* + * Adjusting the prefetch iterator before invoking + * table_scan_bitmap_next_block() keeps prefetch distance higher across + * the parallel workers. + */ + if (scan->prefetch_maximum > 0) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages > 0) + { + pstate->prefetch_pages--; + SpinLockRelease(&pstate->mutex); + } + else + { + /* Release the mutex before iterating */ + SpinLockRelease(&pstate->mutex); + + /* + * In case of shared mode, we can not ensure that the current + * blockno of the main iterator and that of the prefetch iterator + * are same. It's possible that whatever blockno we are + * prefetching will be processed by another process. Therefore, + * we don't validate the blockno here as we do in non-parallel + * case. + */ + if (!prefetch_iterator->exhausted) + { + tbmpre = bhs_iterate(prefetch_iterator); + scan->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; + } + } + } +#endif /* USE_PREFETCH */ +} + static bool -heapam_scan_bitmap_next_block(TableScanDesc scan, - BlockNumber *blockno, bool *recheck, +heapam_scan_bitmap_next_block(TableScanDesc sscan, + bool *recheck, long *lossy_pages, long *exact_pages) { - HeapScanDesc hscan = (HeapScanDesc) scan; + BitmapHeapScanDesc scan = (BitmapHeapScanDesc) sscan; + HeapScanDesc hscan = &scan->heap_common; BlockNumber block; Buffer buffer; Snapshot snapshot; @@ -2201,19 +2272,21 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, hscan->rs_cindex = 0; hscan->rs_ntuples = 0; - *blockno = InvalidBlockNumber; *recheck = true; + BitmapAdjustPrefetchIterator(scan); + do { CHECK_FOR_INTERRUPTS(); - tbmres = bhs_iterate(&scan->rs_bhs_iterator); + Assert(!scan->iterator.exhausted); + tbmres = bhs_iterate(&scan->iterator); if (tbmres == NULL) { /* no more entries in the bitmap */ - Assert(hscan->rs_empty_tuples_pending == 0); + Assert(scan->empty_tuples_pending == 0); return false; } @@ -2228,7 +2301,6 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, } while (!IsolationIsSerializable() && tbmres->blockno >= hscan->rs_nblocks); /* Got a valid block */ - *blockno = tbmres->blockno; *recheck = tbmres->recheck; /* @@ -2236,15 +2308,15 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, * heap, the bitmap entries don't need rechecking, and all tuples on the * page are visible to our transaction. */ - if (!(scan->rs_flags & SO_NEED_TUPLE) && + if (!(hscan->rs_base.rs_flags & SO_NEED_TUPLE) && !tbmres->recheck && - VM_ALL_VISIBLE(scan->rs_rd, tbmres->blockno, &hscan->rs_vmbuffer)) + VM_ALL_VISIBLE(hscan->rs_base.rs_rd, tbmres->blockno, &scan->vmbuffer)) { /* can't be lossy in the skip_fetch case */ Assert(tbmres->ntuples >= 0); - Assert(hscan->rs_empty_tuples_pending >= 0); + Assert(scan->empty_tuples_pending >= 0); - hscan->rs_empty_tuples_pending += tbmres->ntuples; + scan->empty_tuples_pending += tbmres->ntuples; return true; } @@ -2255,18 +2327,18 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, * Acquire pin on the target heap page, trading in any pin we held before. */ hscan->rs_cbuf = ReleaseAndReadBuffer(hscan->rs_cbuf, - scan->rs_rd, + hscan->rs_base.rs_rd, block); hscan->rs_cblock = block; buffer = hscan->rs_cbuf; - snapshot = scan->rs_snapshot; + snapshot = hscan->rs_base.rs_snapshot; ntup = 0; /* * Prune and repair fragmentation for the whole page, if possible. */ - heap_page_prune_opt(scan->rs_rd, buffer); + heap_page_prune_opt(hscan->rs_base.rs_rd, buffer); /* * We must hold share lock on the buffer content while examining tuple @@ -2294,7 +2366,7 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, HeapTupleData heapTuple; ItemPointerSet(&tid, block, offnum); - if (heap_hot_search_buffer(&tid, scan->rs_rd, buffer, snapshot, + if (heap_hot_search_buffer(&tid, hscan->rs_base.rs_rd, buffer, snapshot, &heapTuple, NULL, true)) hscan->rs_vistuples[ntup++] = ItemPointerGetOffsetNumber(&tid); } @@ -2320,16 +2392,16 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, continue; loctup.t_data = (HeapTupleHeader) PageGetItem(page, lp); loctup.t_len = ItemIdGetLength(lp); - loctup.t_tableOid = scan->rs_rd->rd_id; + loctup.t_tableOid = hscan->rs_base.rs_rd->rd_id; ItemPointerSet(&loctup.t_self, block, offnum); valid = HeapTupleSatisfiesVisibility(&loctup, snapshot, buffer); if (valid) { hscan->rs_vistuples[ntup++] = offnum; - PredicateLockTID(scan->rs_rd, &loctup.t_self, snapshot, + PredicateLockTID(hscan->rs_base.rs_rd, &loctup.t_self, snapshot, HeapTupleHeaderGetXmin(loctup.t_data)); } - HeapCheckForSerializableConflictOut(valid, scan->rs_rd, &loctup, + HeapCheckForSerializableConflictOut(valid, hscan->rs_base.rs_rd, &loctup, buffer, snapshot); } } @@ -2344,6 +2416,18 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, else (*exact_pages)++; + /* + * If serial, we can error out if the the prefetch block doesn't stay + * ahead of the current block. + */ + if (scan->pstate == NULL && + !scan->pf_iterator.exhausted && + scan->pfblockno < block) + elog(ERROR, "prefetch and main iterators are out of sync"); + + /* Adjust the prefetch target */ + BitmapAdjustPrefetchTarget(scan); + /* * Return true to indicate that a valid block was found and the bitmap is * not exhausted. If there are no visible tuples on this page, @@ -2354,22 +2438,168 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, return true; } +/* + * BitmapAdjustPrefetchTarget - Adjust the prefetch target + * + * Increase prefetch target if it's not yet at the max. Note that + * we will increase it to zero after fetching the very first + * page/tuple, then to one after the second tuple is fetched, then + * it doubles as later pages are fetched. + */ +static inline void +BitmapAdjustPrefetchTarget(BitmapHeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->pstate; + int prefetch_maximum = scan->prefetch_maximum; + + if (pstate == NULL) + { + if (scan->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (scan->prefetch_target >= prefetch_maximum / 2) + scan->prefetch_target = prefetch_maximum; + else if (scan->prefetch_target > 0) + scan->prefetch_target *= 2; + else + scan->prefetch_target++; + return; + } + + /* Do an unlocked check first to save spinlock acquisitions. */ + if (pstate->prefetch_target < prefetch_maximum) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (pstate->prefetch_target >= prefetch_maximum / 2) + pstate->prefetch_target = prefetch_maximum; + else if (pstate->prefetch_target > 0) + pstate->prefetch_target *= 2; + else + pstate->prefetch_target++; + SpinLockRelease(&pstate->mutex); + } +#endif /* USE_PREFETCH */ +} + + +/* + * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target + */ +static inline void +BitmapPrefetch(BitmapHeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->pstate; + BitmapHeapIterator *prefetch_iterator = &scan->pf_iterator; + Relation rel = scan->heap_common.rs_base.rs_rd; + + if (pstate == NULL) + { + if (!prefetch_iterator->exhausted) + { + while (scan->prefetch_pages < scan->prefetch_target) + { + TBMIterateResult *tbmpre = bhs_iterate(prefetch_iterator); + bool skip_fetch; + + if (tbmpre == NULL) + { + /* No more pages to prefetch */ + bhs_end_iterate(prefetch_iterator); + break; + } + scan->prefetch_pages++; + scan->pfblockno = tbmpre->blockno; + + /* + * If we expect not to have to actually read this heap page, + * skip this prefetch call, but continue to run the prefetch + * logic normally. (Would it be better not to increment + * prefetch_pages?) + */ + skip_fetch = (!(scan->heap_common.rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre->recheck && + VM_ALL_VISIBLE(rel, + tbmpre->blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(rel, MAIN_FORKNUM, tbmpre->blockno); + } + } + + return; + } + + if (pstate->prefetch_pages < pstate->prefetch_target) + { + if (!prefetch_iterator->exhausted) + { + while (1) + { + TBMIterateResult *tbmpre; + bool do_prefetch = false; + bool skip_fetch; + + /* + * Recheck under the mutex. If some other process has already + * done enough prefetching then we need not to do anything. + */ + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages < pstate->prefetch_target) + { + pstate->prefetch_pages++; + do_prefetch = true; + } + SpinLockRelease(&pstate->mutex); + + if (!do_prefetch) + return; + + tbmpre = bhs_iterate(prefetch_iterator); + if (tbmpre == NULL) + { + /* No more pages to prefetch */ + bhs_end_iterate(prefetch_iterator); + break; + } + + scan->pfblockno = tbmpre->blockno; + + /* As above, skip prefetch if we expect not to need page */ + skip_fetch = (!(scan->heap_common.rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre->recheck && + VM_ALL_VISIBLE(rel, + tbmpre->blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(rel, MAIN_FORKNUM, tbmpre->blockno); + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_tuple(TableScanDesc scan, TupleTableSlot *slot) { - HeapScanDesc hscan = (HeapScanDesc) scan; + BitmapHeapScanDesc bscan = (BitmapHeapScanDesc) scan; + HeapScanDesc hscan = &bscan->heap_common; OffsetNumber targoffset; Page page; ItemId lp; - if (hscan->rs_empty_tuples_pending > 0) + if (bscan->empty_tuples_pending > 0) { /* * If we don't have to fetch the tuple, just return nulls. */ ExecStoreAllNullTuple(slot); - hscan->rs_empty_tuples_pending--; + bscan->empty_tuples_pending--; return true; } @@ -2379,6 +2609,36 @@ heapam_scan_bitmap_next_tuple(TableScanDesc scan, if (hscan->rs_cindex < 0 || hscan->rs_cindex >= hscan->rs_ntuples) return false; +#ifdef USE_PREFETCH + + /* + * Try to prefetch at least a few pages even before we get to the second + * page if we don't stop reading after the first tuple. + */ + if (!bscan->pstate) + { + if (bscan->prefetch_target < bscan->prefetch_maximum) + bscan->prefetch_target++; + } + else if (bscan->pstate->prefetch_target < bscan->prefetch_maximum) + { + /* take spinlock while updating shared state */ + SpinLockAcquire(&bscan->pstate->mutex); + if (bscan->pstate->prefetch_target < bscan->prefetch_maximum) + bscan->pstate->prefetch_target++; + SpinLockRelease(&bscan->pstate->mutex); + } + + /* + * We issue prefetch requests *after* fetching the current page to try to + * avoid having prefetching interfere with the main I/O. Also, this should + * happen only when we have determined there is still something to do on + * the current page, else we may uselessly prefetch the same page we are + * just about to request for real. + */ + BitmapPrefetch(bscan); +#endif /* USE_PREFETCH */ + targoffset = hscan->rs_vistuples[hscan->rs_cindex]; page = BufferGetPage(hscan->rs_cbuf); lp = PageGetItemId(page, targoffset); @@ -2704,6 +2964,10 @@ static const TableAmRoutine heapam_methods = { .scan_set_tidrange = heap_set_tidrange, .scan_getnextslot_tidrange = heap_getnextslot_tidrange, + .scan_rescan_bm = heap_rescan_bm, + .scan_begin_bm = heap_beginscan_bm, + .scan_end_bm = heap_endscan_bm, + .parallelscan_estimate = table_block_parallelscan_estimate, .parallelscan_initialize = table_block_parallelscan_initialize, .parallelscan_reinitialize = table_block_parallelscan_reinitialize, diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index 70b560b8ee..d22a433f06 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -51,10 +51,6 @@ static TupleTableSlot *BitmapHeapNext(BitmapHeapScanState *node); static inline void BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate); -static inline void BitmapAdjustPrefetchIterator(BitmapHeapScanState *node); -static inline void BitmapAdjustPrefetchTarget(BitmapHeapScanState *node); -static inline void BitmapPrefetch(BitmapHeapScanState *node, - TableScanDesc scan); static bool BitmapShouldInitializeSharedState(ParallelBitmapHeapState *pstate); /* @@ -122,20 +118,9 @@ bhs_end_iterate(BitmapHeapIterator *iterator) static TupleTableSlot * BitmapHeapNext(BitmapHeapScanState *node) { - ExprContext *econtext; - TableScanDesc scan; - TIDBitmap *tbm; - TupleTableSlot *slot; - ParallelBitmapHeapState *pstate = node->pstate; - dsa_area *dsa = node->ss.ps.state->es_query_dsa; - - /* - * extract necessary information from index scan node - */ - econtext = node->ss.ps.ps_ExprContext; - slot = node->ss.ss_ScanTupleSlot; - scan = node->ss.ss_currentScanDesc; - tbm = node->tbm; + ExprContext *econtext = node->ss.ps.ps_ExprContext; + TupleTableSlot *slot = node->ss.ss_ScanTupleSlot; + TableScanDesc scan = node->ss.ss_currentScanDesc; /* * If we haven't yet performed the underlying index scan, do it, and begin @@ -146,25 +131,37 @@ BitmapHeapNext(BitmapHeapScanState *node) * prefetching. node->prefetch_pages tracks exactly how many pages ahead * the prefetch iterator is. Also, node->prefetch_target tracks the * desired prefetch distance, which starts small and increases up to the - * node->prefetch_maximum. This is to avoid doing a lot of prefetching in + * scan->prefetch_maximum. This is to avoid doing a lot of prefetching in * a scan that stops after a few tuples because of a LIMIT. */ if (!node->initialized) { + Relation rel = node->ss.ss_currentRelation; + bool pf_maximum = 0; + bool init_shared_state = false; + uint32 extra_flags = 0; + /* * The leader will immediately come out of the function, but others * will be blocked until leader populates the TBM and wakes them up. */ - bool init_shared_state = node->pstate ? + init_shared_state = node->pstate ? BitmapShouldInitializeSharedState(node->pstate) : false; - if (!pstate || init_shared_state) + /* + * Maximum number of prefetches for the tablespace if configured, + * otherwise the current value of the effective_io_concurrency GUC. + */ +#ifdef USE_PREFETCH + pf_maximum = get_tablespace_io_concurrency(rel->rd_rel->reltablespace); +#endif + + if (!node->pstate || init_shared_state) { - tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); + node->tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); - if (!tbm || !IsA(tbm, TIDBitmap)) + if (!node->tbm || !IsA(node->tbm, TIDBitmap)) elog(ERROR, "unrecognized result from subplan"); - node->tbm = tbm; if (init_shared_state) { @@ -173,113 +170,52 @@ BitmapHeapNext(BitmapHeapScanState *node) * dsa_pointer of the iterator state which will be used by * multiple processes to iterate jointly. */ - pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); + node->pstate->tbmiterator = tbm_prepare_shared_iterate(node->tbm); + #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (pf_maximum > 0) { - pstate->prefetch_iterator = - tbm_prepare_shared_iterate(tbm); - - /* - * We don't need the mutex here as we haven't yet woke up - * others. - */ - pstate->prefetch_pages = 0; - pstate->prefetch_target = -1; + node->pstate->prefetch_iterator = + tbm_prepare_shared_iterate(node->tbm); } #endif /* We have initialized the shared state so wake up others. */ - BitmapDoneInitializingSharedState(pstate); + BitmapDoneInitializingSharedState(node->pstate); } } /* - * If this is the first scan of the underlying table, create the table - * scan descriptor and begin the scan. + * We can potentially skip fetching heap pages if we do not need any + * columns of the table, either for checking non-indexable quals or + * for returning data. This test is a bit simplistic, as it checks + * the stronger condition that there's no qual or return tlist at all. + * But in most cases it's probably not worth working harder than that. */ - if (!scan) - { - uint32 extra_flags = 0; - - /* - * We can potentially skip fetching heap pages if we do not need - * any columns of the table, either for checking non-indexable - * quals or for returning data. This test is a bit simplistic, as - * it checks the stronger condition that there's no qual or return - * tlist at all. But in most cases it's probably not worth working - * harder than that. - */ - if (node->ss.ps.plan->qual != NIL || node->ss.ps.plan->targetlist != NIL) - extra_flags |= SO_NEED_TUPLE; - - scan = table_beginscan_bm(node->ss.ss_currentRelation, - node->ss.ps.state->es_snapshot, - 0, - NULL, - extra_flags); - - node->ss.ss_currentScanDesc = scan; - } - - bhs_begin_iterate(&scan->rs_bhs_iterator, tbm, dsa, - pstate ? - pstate->tbmiterator : - InvalidDsaPointer); - -#ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) - { - bhs_begin_iterate(&node->pf_iterator, tbm, dsa, - pstate ? - pstate->prefetch_iterator : - InvalidDsaPointer); - /* Only used for serial BHS */ - node->prefetch_pages = 0; - node->prefetch_target = -1; - } -#endif /* USE_PREFETCH */ - + if (node->ss.ps.plan->qual != NIL || node->ss.ps.plan->targetlist != NIL) + extra_flags |= SO_NEED_TUPLE; + + scan = table_beginscan_bm(node->ss.ss_currentScanDesc, + rel, + node->ss.ps.state->es_snapshot, + extra_flags, + pf_maximum, + node->tbm, + node->pstate, + node->ss.ps.state->es_query_dsa); + + node->ss.ss_currentScanDesc = scan; node->initialized = true; goto new_page; } + for (;;) { while (table_scan_bitmap_next_tuple(scan, slot)) { CHECK_FOR_INTERRUPTS(); -#ifdef USE_PREFETCH - - /* - * Try to prefetch at least a few pages even before we get to the - * second page if we don't stop reading after the first tuple. - */ - if (!pstate) - { - if (node->prefetch_target < node->prefetch_maximum) - node->prefetch_target++; - } - else if (pstate->prefetch_target < node->prefetch_maximum) - { - /* take spinlock while updating shared state */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target < node->prefetch_maximum) - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ - - /* - * We issue prefetch requests *after* fetching the current page to - * try to avoid having prefetching interfere with the main I/O. - * Also, this should happen only when we have determined there is - * still something to do on the current page, else we may - * uselessly prefetch the same page we are just about to request - * for real. - */ - BitmapPrefetch(node, scan); /* * If we are using lossy info, we have to recheck the qual @@ -303,23 +239,9 @@ BitmapHeapNext(BitmapHeapScanState *node) new_page: - BitmapAdjustPrefetchIterator(node); - - if (!table_scan_bitmap_next_block(scan, &node->blockno, &node->recheck, + if (!table_scan_bitmap_next_block(scan, &node->recheck, &node->lossy_pages, &node->exact_pages)) break; - - /* - * If serial, we can error out if the the prefetch block doesn't stay - * ahead of the current block. - */ - if (node->pstate == NULL && - !node->pf_iterator.exhausted && - node->pfblockno < node->blockno) - elog(ERROR, "prefetch and main iterators are out of sync"); - - /* Adjust the prefetch target */ - BitmapAdjustPrefetchTarget(node); } /* @@ -343,215 +265,6 @@ BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate) ConditionVariableBroadcast(&pstate->cv); } -/* - * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator - * - * We keep track of how far the prefetch iterator is ahead of the main - * iterator in prefetch_pages. For each block the main iterator returns, we - * decrement prefetch_pages. - */ -static inline void -BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - BitmapHeapIterator *prefetch_iterator = &node->pf_iterator; - TBMIterateResult *tbmpre; - - if (pstate == NULL) - { - if (node->prefetch_pages > 0) - { - /* The main iterator has closed the distance by one page */ - node->prefetch_pages--; - } - else if (!prefetch_iterator->exhausted) - { - /* Do not let the prefetch iterator get behind the main one */ - tbmpre = bhs_iterate(prefetch_iterator); - node->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; - } - return; - } - - /* - * Adjusting the prefetch iterator before invoking - * table_scan_bitmap_next_block() keeps prefetch distance higher across - * the parallel workers. - */ - if (node->prefetch_maximum > 0) - { - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages > 0) - { - pstate->prefetch_pages--; - SpinLockRelease(&pstate->mutex); - } - else - { - /* Release the mutex before iterating */ - SpinLockRelease(&pstate->mutex); - - /* - * In case of shared mode, we can not ensure that the current - * blockno of the main iterator and that of the prefetch iterator - * are same. It's possible that whatever blockno we are - * prefetching will be processed by another process. Therefore, - * we don't validate the blockno here as we do in non-parallel - * case. - */ - if (!prefetch_iterator->exhausted) - { - tbmpre = bhs_iterate(prefetch_iterator); - node->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; - } - } - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapAdjustPrefetchTarget - Adjust the prefetch target - * - * Increase prefetch target if it's not yet at the max. Note that - * we will increase it to zero after fetching the very first - * page/tuple, then to one after the second tuple is fetched, then - * it doubles as later pages are fetched. - */ -static inline void -BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - - if (pstate == NULL) - { - if (node->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (node->prefetch_target >= node->prefetch_maximum / 2) - node->prefetch_target = node->prefetch_maximum; - else if (node->prefetch_target > 0) - node->prefetch_target *= 2; - else - node->prefetch_target++; - return; - } - - /* Do an unlocked check first to save spinlock acquisitions. */ - if (pstate->prefetch_target < node->prefetch_maximum) - { - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (pstate->prefetch_target >= node->prefetch_maximum / 2) - pstate->prefetch_target = node->prefetch_maximum; - else if (pstate->prefetch_target > 0) - pstate->prefetch_target *= 2; - else - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target - */ -static inline void -BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - BitmapHeapIterator *prefetch_iterator = &node->pf_iterator; - - if (pstate == NULL) - { - if (!prefetch_iterator->exhausted) - { - while (node->prefetch_pages < node->prefetch_target) - { - TBMIterateResult *tbmpre = bhs_iterate(prefetch_iterator); - bool skip_fetch; - - if (tbmpre == NULL) - { - /* No more pages to prefetch */ - bhs_end_iterate(prefetch_iterator); - break; - } - node->prefetch_pages++; - node->pfblockno = tbmpre->blockno; - - /* - * If we expect not to have to actually read this heap page, - * skip this prefetch call, but continue to run the prefetch - * logic normally. (Would it be better not to increment - * prefetch_pages?) - */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre->recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre->blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre->blockno); - } - } - - return; - } - - if (pstate->prefetch_pages < pstate->prefetch_target) - { - if (!prefetch_iterator->exhausted) - { - while (1) - { - TBMIterateResult *tbmpre; - bool do_prefetch = false; - bool skip_fetch; - - /* - * Recheck under the mutex. If some other process has already - * done enough prefetching then we need not to do anything. - */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages < pstate->prefetch_target) - { - pstate->prefetch_pages++; - do_prefetch = true; - } - SpinLockRelease(&pstate->mutex); - - if (!do_prefetch) - return; - - tbmpre = bhs_iterate(prefetch_iterator); - if (tbmpre == NULL) - { - /* No more pages to prefetch */ - bhs_end_iterate(prefetch_iterator); - break; - } - - node->pfblockno = tbmpre->blockno; - - /* As above, skip prefetch if we expect not to need page */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre->recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre->blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre->blockno); - } - } - } -#endif /* USE_PREFETCH */ -} - /* * BitmapHeapRecheck -- access method routine to recheck a tuple in EvalPlanQual */ @@ -597,19 +310,12 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) if (node->ss.ss_currentScanDesc) table_rescan(node->ss.ss_currentScanDesc, NULL); - /* release bitmaps and buffers if any */ - if (node->pf_iterator.exhausted) - bhs_end_iterate(&node->pf_iterator); + /* release bitmaps if any */ if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); node->tbm = NULL; node->initialized = false; - node->pvmbuffer = InvalidBuffer; node->recheck = true; - node->blockno = InvalidBlockNumber; - node->pfblockno = InvalidBlockNumber; ExecScanReScan(&node->ss); @@ -648,14 +354,10 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) table_endscan(scanDesc); /* - * release bitmaps and buffers if any + * release bitmaps if any */ - if (!node->pf_iterator.exhausted) - bhs_end_iterate(&node->pf_iterator); if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); } /* ---------------------------------------------------------------- @@ -688,16 +390,11 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ps.ExecProcNode = ExecBitmapHeapScan; scanstate->tbm = NULL; - scanstate->pvmbuffer = InvalidBuffer; scanstate->exact_pages = 0; scanstate->lossy_pages = 0; - scanstate->prefetch_pages = 0; - scanstate->prefetch_target = 0; scanstate->initialized = false; scanstate->pstate = NULL; scanstate->recheck = true; - scanstate->blockno = InvalidBlockNumber; - scanstate->pfblockno = InvalidBlockNumber; /* * Miscellaneous initialization @@ -737,13 +434,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->bitmapqualorig = ExecInitQual(node->bitmapqualorig, (PlanState *) scanstate); - /* - * Maximum number of prefetches for the tablespace if configured, - * otherwise the current value of the effective_io_concurrency GUC. - */ - scanstate->prefetch_maximum = - get_tablespace_io_concurrency(currentRelation->rd_rel->reltablespace); - scanstate->ss.ss_currentRelation = currentRelation; /* @@ -827,7 +517,7 @@ ExecBitmapHeapInitializeDSM(BitmapHeapScanState *node, /* Initialize the mutex */ SpinLockInit(&pstate->mutex); pstate->prefetch_pages = 0; - pstate->prefetch_target = 0; + pstate->prefetch_target = -1; pstate->state = BM_INITIAL; ConditionVariableInit(&pstate->cv); diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 750ea30852..73690d15c5 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -76,22 +76,60 @@ typedef struct HeapScanDescData */ ParallelBlockTableScanWorkerData *rs_parallelworkerdata; + /* these fields only used in page-at-a-time mode and for bitmap scans */ + int rs_cindex; /* current tuple's index in vistuples */ + int rs_ntuples; /* number of visible tuples on page */ + OffsetNumber rs_vistuples[MaxHeapTuplesPerPage]; /* their offsets */ + +} HeapScanDescData; +typedef struct HeapScanDescData *HeapScanDesc; + +typedef struct BitmapHeapScanDescData +{ + /* All the non-BitmapHeapScan specific members */ + HeapScanDescData heap_common; + + /* + * Members common to Parallel and Serial BitmapHeapScan + */ + struct BitmapHeapIterator iterator; + struct BitmapHeapIterator pf_iterator; + + /* maximum value for prefetch_target */ + int prefetch_maximum; + /* * These fields are only used for bitmap scans for the "skip fetch" * optimization. Bitmap scans needing no fields from the heap may skip * fetching an all visible block, instead using the number of tuples per * block reported by the bitmap to determine how many NULL-filled tuples - * to return. + * to return. They are common to parallel and serial BitmapHeapScans */ - Buffer rs_vmbuffer; - int rs_empty_tuples_pending; + Buffer vmbuffer; + /* buffer for visibility-map lookups of prefetched pages */ + Buffer pvmbuffer; + int empty_tuples_pending; + + /* + * Parallel-only members + */ + + struct ParallelBitmapHeapState *pstate; + + /* + * Serial-only members + */ + + /* Current target for prefetch distance */ + int prefetch_target; + /* # pages prefetch iterator is ahead of current */ + int prefetch_pages; + /* used to validate prefetch block stays ahead of current block */ + BlockNumber pfblockno; +} BitmapHeapScanDescData; + +typedef struct BitmapHeapScanDescData *BitmapHeapScanDesc; - /* these fields only used in page-at-a-time mode and for bitmap scans */ - int rs_cindex; /* current tuple's index in vistuples */ - int rs_ntuples; /* number of visible tuples on page */ - OffsetNumber rs_vistuples[MaxHeapTuplesPerPage]; /* their offsets */ -} HeapScanDescData; -typedef struct HeapScanDescData *HeapScanDesc; /* * Descriptor for fetches from heap via an index. @@ -289,6 +327,12 @@ extern void heap_set_tidrange(TableScanDesc sscan, ItemPointer mintid, extern bool heap_getnextslot_tidrange(TableScanDesc sscan, ScanDirection direction, TupleTableSlot *slot); +extern TableScanDesc heap_beginscan_bm(Relation relation, Snapshot snapshot, uint32 flags); + +extern void heap_endscan_bm(TableScanDesc scan); +extern void heap_rescan_bm(TableScanDesc sscan, TIDBitmap *tbm, + ParallelBitmapHeapState *pstate, dsa_area *dsa, int pf_maximum); + extern bool heap_fetch(Relation relation, Snapshot snapshot, HeapTuple tuple, Buffer *userbuf, bool keep_buf); extern bool heap_hot_search_buffer(ItemPointer tid, Relation relation, diff --git a/src/include/access/relscan.h b/src/include/access/relscan.h index e520186b41..855b9558bf 100644 --- a/src/include/access/relscan.h +++ b/src/include/access/relscan.h @@ -41,9 +41,6 @@ typedef struct TableScanDescData ItemPointerData rs_mintid; ItemPointerData rs_maxtid; - /* Only used for Bitmap table scans */ - BitmapHeapIterator rs_bhs_iterator; - /* * Information about type and behaviour of the scan, a bitmask of members * of the ScanOptions enum (see tableam.h). diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 618ab2b449..138bf5f6ed 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -391,6 +391,23 @@ typedef struct TableAmRoutine ScanDirection direction, TupleTableSlot *slot); + /* + * TODO: add comment + */ + TableScanDesc (*scan_begin_bm) (Relation rel, + Snapshot snapshot, + uint32 flags); + + void (*scan_rescan_bm) (TableScanDesc scan, TIDBitmap *tbm, + ParallelBitmapHeapState *pstate, dsa_area *dsa, + int pf_maximum); + + /* + * Release resources and deallocate scan. If TableScanDesc.temp_snap, + * TableScanDesc.rs_snapshot needs to be unregistered. + */ + void (*scan_end_bm) (TableScanDesc scan); + /* ------------------------------------------------------------------------ * Parallel table scan related functions. * ------------------------------------------------------------------------ @@ -774,9 +791,9 @@ typedef struct TableAmRoutine */ /* - * Prepare to fetch / check / return tuples from `blockno` as part of a - * bitmap table scan. `scan` was started via table_beginscan_bm(). Return - * false if the bitmap is exhausted and true otherwise. + * Prepare to fetch / check / return tuples as part of a bitmap table + * scan. `scan` was started via table_beginscan_bm(). Return false if the + * bitmap is exhausted and true otherwise. * * This will typically read and pin the target block, and do the necessary * work to allow scan_bitmap_next_tuple() to return tuples (e.g. it might @@ -785,22 +802,11 @@ typedef struct TableAmRoutine * lossy_pages is incremented if the bitmap is lossy for the selected * block; otherwise, exact_pages is incremented. * - * XXX: Currently this may only be implemented if the AM uses md.c as its - * storage manager, and uses ItemPointer->ip_blkid in a manner that maps - * blockids directly to the underlying storage. nodeBitmapHeapscan.c - * performs prefetching directly using that interface. This probably - * needs to be rectified at a later point. - * - * XXX: Currently this may only be implemented if the AM uses the - * visibilitymap, as nodeBitmapHeapscan.c unconditionally accesses it to - * perform prefetching. This probably needs to be rectified at a later - * point. - * * Optional callback, but either both scan_bitmap_next_block and * scan_bitmap_next_tuple need to exist, or neither. */ bool (*scan_bitmap_next_block) (TableScanDesc scan, - BlockNumber *blockno, bool *recheck, + bool *recheck, long *lossy_pages, long *exact_pages); /* @@ -929,18 +935,26 @@ table_beginscan_strat(Relation rel, Snapshot snapshot, } /* - * table_beginscan_bm is an alternative entry point for setting up a - * TableScanDesc for a bitmap heap scan. Although that scan technology is - * really quite unlike a standard seqscan, there is just enough commonality to - * make it worth using the same data structure. + * table_beginscan_bm is the entry point for setting up a TableScanDesc for a + * bitmap heap scan. */ static inline TableScanDesc -table_beginscan_bm(Relation rel, Snapshot snapshot, - int nkeys, struct ScanKeyData *key, uint32 extra_flags) +table_beginscan_bm(TableScanDesc scan, Relation rel, Snapshot snapshot, + uint32 extra_flags, int pf_maximum, TIDBitmap *tbm, + ParallelBitmapHeapState *pstate, dsa_area *dsa) { uint32 flags = SO_TYPE_BITMAPSCAN | SO_ALLOW_PAGEMODE | extra_flags; - return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + scan = rel->rd_tableam->scan_begin_bm(rel, snapshot, flags); + + scan->rs_rd->rd_tableam->scan_rescan_bm(scan, tbm, pstate, dsa, pf_maximum); + + return scan; } /* @@ -987,9 +1001,11 @@ table_beginscan_tid(Relation rel, Snapshot snapshot) static inline void table_endscan(TableScanDesc scan) { - if (scan->rs_flags & SO_TYPE_BITMAPSCAN && - !scan->rs_bhs_iterator.exhausted) - bhs_end_iterate(&scan->rs_bhs_iterator); + if (scan->rs_flags & SO_TYPE_BITMAPSCAN) + { + scan->rs_rd->rd_tableam->scan_end_bm(scan); + return; + } scan->rs_rd->rd_tableam->scan_end(scan); } @@ -1001,10 +1017,6 @@ static inline void table_rescan(TableScanDesc scan, struct ScanKeyData *key) { - if (scan->rs_flags & SO_TYPE_BITMAPSCAN && - !scan->rs_bhs_iterator.exhausted) - bhs_end_iterate(&scan->rs_bhs_iterator); - scan->rs_rd->rd_tableam->scan_rescan(scan, key, false, false, false, false); } @@ -1979,7 +1991,7 @@ table_relation_estimate_size(Relation rel, int32 *attr_widths, */ static inline bool table_scan_bitmap_next_block(TableScanDesc scan, - BlockNumber *blockno, bool *recheck, + bool *recheck, long *lossy_pages, long *exact_pages) { @@ -1992,7 +2004,7 @@ table_scan_bitmap_next_block(TableScanDesc scan, elog(ERROR, "unexpected table_scan_bitmap_next_block call during logical decoding"); return scan->rs_rd->rd_tableam->scan_bitmap_next_block(scan, - blockno, recheck, + recheck, lossy_pages, exact_pages); } diff --git a/src/include/executor/nodeBitmapHeapscan.h b/src/include/executor/nodeBitmapHeapscan.h index 7064f54686..f3c92f7a8a 100644 --- a/src/include/executor/nodeBitmapHeapscan.h +++ b/src/include/executor/nodeBitmapHeapscan.h @@ -28,6 +28,12 @@ extern void ExecBitmapHeapReInitializeDSM(BitmapHeapScanState *node, ParallelContext *pcxt); extern void ExecBitmapHeapInitializeWorker(BitmapHeapScanState *node, ParallelWorkerContext *pwcxt); +typedef struct BitmapHeapIterator +{ + struct TBMIterator *serial; + struct TBMSharedIterator *parallel; + bool exhausted; +} BitmapHeapIterator; extern TBMIterateResult *bhs_iterate(BitmapHeapIterator *iterator); extern void bhs_begin_iterate(BitmapHeapIterator *iterator, TIDBitmap *tbm, diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 714eb3e534..49ce4ff9a7 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1787,30 +1787,16 @@ typedef struct ParallelBitmapHeapState ConditionVariable cv; } ParallelBitmapHeapState; -typedef struct BitmapHeapIterator -{ - struct TBMIterator *serial; - struct TBMSharedIterator *parallel; - bool exhausted; -} BitmapHeapIterator; - /* ---------------- * BitmapHeapScanState information * * bitmapqualorig execution state for bitmapqualorig expressions * tbm bitmap obtained from child index scan(s) - * pvmbuffer buffer for visibility-map lookups of prefetched pages * exact_pages total number of exact pages retrieved * lossy_pages total number of lossy pages retrieved - * pf_iterator for prefetching ahead of current page - * prefetch_pages # pages prefetch iterator is ahead of current - * prefetch_target current target prefetch distance - * prefetch_maximum maximum value for prefetch_target * initialized is node is ready to iterate * pstate shared state for parallel bitmap scan * recheck do current page's tuples need recheck - * blockno used to validate pf and current block in sync - * pfblockno used to validate pf stays ahead of current block * ---------------- */ typedef struct BitmapHeapScanState @@ -1818,18 +1804,11 @@ typedef struct BitmapHeapScanState ScanState ss; /* its first field is NodeTag */ ExprState *bitmapqualorig; TIDBitmap *tbm; - Buffer pvmbuffer; long exact_pages; long lossy_pages; - int prefetch_pages; - int prefetch_target; - int prefetch_maximum; bool initialized; - BitmapHeapIterator pf_iterator; ParallelBitmapHeapState *pstate; bool recheck; - BlockNumber blockno; - BlockNumber pfblockno; } BitmapHeapScanState; /* ---------------- diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 2348a8793e..443b01c053 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -263,6 +263,7 @@ BitmapHeapIterator BitmapHeapPath BitmapHeapScan BitmapHeapScanState +BitmapHeapScanDesc BitmapIndexScan BitmapIndexScanState BitmapOr -- 2.40.1 --pbix6fw3h4kvmjae Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v15-0012-Move-BitmapHeapScan-initialization-to-helper.patch" ^ permalink raw reply [nested|flat] 10+ messages in thread
* [PATCH v10 14/17] Push BitmapHeapScan prefetch code into heapam.c @ 2024-03-22 13:42 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 10+ messages in thread From: Melanie Plageman @ 2024-03-22 13:42 UTC (permalink / raw) In preparation for transitioning to using the streaming read API for prefetching [1], move all of the BitmapHeapScanState members related to prefetching and the functions for accessing them into the HeapScanDescData and TableScanDescData. Members that still need to be accessed in BitmapHeapNext() could not be moved into heap AM-specific code. Specifically, parallel iterator setup requires several components which seem odd to pass to the table AM API. [1] https://www.postgresql.org/message-id/flat/CA%2BhUKGJkOiOCa%2Bmag4BF%2BzHo7qo%3Do9CFheB8%3Dg6uT5TUm2... --- src/backend/access/heap/heapam.c | 26 ++ src/backend/access/heap/heapam_handler.c | 262 +++++++++++++++++ src/backend/executor/nodeBitmapHeapscan.c | 341 ++-------------------- src/include/access/heapam.h | 17 ++ src/include/access/relscan.h | 8 + src/include/access/tableam.h | 26 +- src/include/nodes/execnodes.h | 14 - 7 files changed, 355 insertions(+), 339 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 6b2863391f..3d92fb5135 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -948,8 +948,16 @@ heap_beginscan(Relation relation, Snapshot snapshot, scan->rs_base.rs_flags = flags; scan->rs_base.rs_parallel = parallel_scan; scan->rs_strategy = NULL; /* set in initscan */ + + scan->rs_base.blockno = InvalidBlockNumber; + scan->rs_vmbuffer = InvalidBuffer; scan->rs_empty_tuples_pending = 0; + scan->pvmbuffer = InvalidBuffer; + + scan->pfblockno = InvalidBlockNumber; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; /* * Disable page-at-a-time mode if it's not a MVCC-safe snapshot. @@ -1032,6 +1040,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, scan->rs_base.rs_flags &= ~SO_ALLOW_PAGEMODE; } + scan->rs_base.blockno = InvalidBlockNumber; + + scan->pfblockno = InvalidBlockNumber; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; + /* * unpin scan buffers */ @@ -1044,6 +1058,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, scan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(scan->pvmbuffer)) + { + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + } + /* * reinitialize scan descriptor */ @@ -1069,6 +1089,12 @@ heap_endscan(TableScanDesc sscan) scan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(scan->pvmbuffer)) + { + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + } + /* * decrement relation reference count and free scan descriptor storage */ diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 105137396b..867061325c 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -55,6 +55,9 @@ static bool SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, OffsetNumber tupoffset); static BlockNumber heapam_scan_get_blocks_done(HeapScanDesc hscan); +static inline void BitmapAdjustPrefetchIterator(HeapScanDesc scan); +static inline void BitmapAdjustPrefetchTarget(HeapScanDesc scan); +static inline void BitmapPrefetch(HeapScanDesc scan); static const TableAmRoutine heapam_methods; @@ -2112,6 +2115,73 @@ heapam_estimate_rel_size(Relation rel, int32 *attr_widths, * ------------------------------------------------------------------------ */ +/* + * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator + * + * We keep track of how far the prefetch iterator is ahead of the main + * iterator in prefetch_pages. For each block the main iterator returns, we + * decrement prefetch_pages. + */ +static inline void +BitmapAdjustPrefetchIterator(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + BitmapHeapIterator *prefetch_iterator = scan->rs_base.rs_pf_bhs_iterator; + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + TBMIterateResult tbmpre; + + if (pstate == NULL) + { + if (scan->prefetch_pages > 0) + { + /* The main iterator has closed the distance by one page */ + scan->prefetch_pages--; + } + else if (prefetch_iterator) + { + /* Do not let the prefetch iterator get behind the main one */ + bhs_iterate(prefetch_iterator, &tbmpre); + scan->pfblockno = tbmpre.blockno; + } + return; + } + + /* + * Adjusting the prefetch iterator before invoking + * table_scan_bitmap_next_block() keeps prefetch distance higher across + * the parallel workers. + */ + if (scan->rs_base.prefetch_maximum > 0) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages > 0) + { + pstate->prefetch_pages--; + SpinLockRelease(&pstate->mutex); + } + else + { + /* Release the mutex before iterating */ + SpinLockRelease(&pstate->mutex); + + /* + * In case of shared mode, we can not ensure that the current + * blockno of the main iterator and that of the prefetch iterator + * are same. It's possible that whatever blockno we are + * prefetching will be processed by another process. Therefore, + * we don't validate the blockno here as we do in non-parallel + * case. + */ + if (prefetch_iterator) + { + bhs_iterate(prefetch_iterator, &tbmpre); + scan->pfblockno = tbmpre.blockno; + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_block(TableScanDesc scan, bool *recheck, BlockNumber *blockno, @@ -2130,6 +2200,8 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, *blockno = InvalidBlockNumber; *recheck = true; + BitmapAdjustPrefetchIterator(hscan); + do { CHECK_FOR_INTERRUPTS(); @@ -2270,6 +2342,18 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, else (*exact_pages)++; + /* + * If serial, we can error out if the the prefetch block doesn't stay + * ahead of the current block. + */ + if (scan->bm_parallel == NULL && + scan->rs_pf_bhs_iterator && + hscan->pfblockno > hscan->rs_base.blockno) + elog(ERROR, "prefetch and main iterators are out of sync"); + + /* Adjust the prefetch target */ + BitmapAdjustPrefetchTarget(hscan); + /* * Return true to indicate that a valid block was found and the bitmap is * not exhausted. If there are no visible tuples on this page, @@ -2280,6 +2364,154 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, return true; } +/* + * BitmapAdjustPrefetchTarget - Adjust the prefetch target + * + * Increase prefetch target if it's not yet at the max. Note that + * we will increase it to zero after fetching the very first + * page/tuple, then to one after the second tuple is fetched, then + * it doubles as later pages are fetched. + */ +static inline void +BitmapAdjustPrefetchTarget(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + int prefetch_maximum = scan->rs_base.prefetch_maximum; + + if (pstate == NULL) + { + if (scan->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (scan->prefetch_target >= prefetch_maximum / 2) + scan->prefetch_target = prefetch_maximum; + else if (scan->prefetch_target > 0) + scan->prefetch_target *= 2; + else + scan->prefetch_target++; + return; + } + + /* Do an unlocked check first to save spinlock acquisitions. */ + if (pstate->prefetch_target < prefetch_maximum) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (pstate->prefetch_target >= prefetch_maximum / 2) + pstate->prefetch_target = prefetch_maximum; + else if (pstate->prefetch_target > 0) + pstate->prefetch_target *= 2; + else + pstate->prefetch_target++; + SpinLockRelease(&pstate->mutex); + } +#endif /* USE_PREFETCH */ +} + + +/* + * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target + */ +static inline void +BitmapPrefetch(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + BitmapHeapIterator *prefetch_iterator = scan->rs_base.rs_pf_bhs_iterator; + + if (pstate == NULL) + { + if (prefetch_iterator) + { + while (scan->prefetch_pages < scan->prefetch_target) + { + TBMIterateResult tbmpre; + bool skip_fetch; + + bhs_iterate(prefetch_iterator, &tbmpre); + + if (!BlockNumberIsValid(tbmpre.blockno)) + { + /* No more pages to prefetch */ + bhs_end_iterate(prefetch_iterator); + scan->rs_base.rs_pf_bhs_iterator = NULL; + break; + } + scan->prefetch_pages++; + scan->pfblockno = tbmpre.blockno; + + /* + * If we expect not to have to actually read this heap page, + * skip this prefetch call, but continue to run the prefetch + * logic normally. (Would it be better not to increment + * prefetch_pages?) + */ + skip_fetch = (!(scan->rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre.recheck && + VM_ALL_VISIBLE(scan->rs_base.rs_rd, + tbmpre.blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(scan->rs_base.rs_rd, MAIN_FORKNUM, tbmpre.blockno); + } + } + + return; + } + + if (pstate->prefetch_pages < pstate->prefetch_target) + { + if (prefetch_iterator) + { + while (1) + { + TBMIterateResult tbmpre; + bool do_prefetch = false; + bool skip_fetch; + + /* + * Recheck under the mutex. If some other process has already + * done enough prefetching then we need not to do anything. + */ + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages < pstate->prefetch_target) + { + pstate->prefetch_pages++; + do_prefetch = true; + } + SpinLockRelease(&pstate->mutex); + + if (!do_prefetch) + return; + + bhs_iterate(prefetch_iterator, &tbmpre); + if (!BlockNumberIsValid(tbmpre.blockno)) + { + /* No more pages to prefetch */ + bhs_end_iterate(prefetch_iterator); + scan->rs_base.rs_pf_bhs_iterator = NULL; + break; + } + + scan->pfblockno = tbmpre.blockno; + + /* As above, skip prefetch if we expect not to need page */ + skip_fetch = (!(scan->rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre.recheck && + VM_ALL_VISIBLE(scan->rs_base.rs_rd, + tbmpre.blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(scan->rs_base.rs_rd, MAIN_FORKNUM, tbmpre.blockno); + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_tuple(TableScanDesc scan, TupleTableSlot *slot) @@ -2305,6 +2537,36 @@ heapam_scan_bitmap_next_tuple(TableScanDesc scan, if (hscan->rs_cindex < 0 || hscan->rs_cindex >= hscan->rs_ntuples) return false; +#ifdef USE_PREFETCH + + /* + * Try to prefetch at least a few pages even before we get to the second + * page if we don't stop reading after the first tuple. + */ + if (!scan->bm_parallel) + { + if (hscan->prefetch_target < scan->prefetch_maximum) + hscan->prefetch_target++; + } + else if (scan->bm_parallel->prefetch_target < scan->prefetch_maximum) + { + /* take spinlock while updating shared state */ + SpinLockAcquire(&scan->bm_parallel->mutex); + if (scan->bm_parallel->prefetch_target < scan->prefetch_maximum) + scan->bm_parallel->prefetch_target++; + SpinLockRelease(&scan->bm_parallel->mutex); + } + + /* + * We issue prefetch requests *after* fetching the current page to try to + * avoid having prefetching interfere with the main I/O. Also, this should + * happen only when we have determined there is still something to do on + * the current page, else we may uselessly prefetch the same page we are + * just about to request for real. + */ + BitmapPrefetch(hscan); +#endif /* USE_PREFETCH */ + targoffset = hscan->rs_vistuples[hscan->rs_cindex]; page = BufferGetPage(hscan->rs_cbuf); lp = PageGetItemId(page, targoffset); diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index 78f79aafff..187b288e68 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -51,10 +51,6 @@ static TupleTableSlot *BitmapHeapNext(BitmapHeapScanState *node); static inline void BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate); -static inline void BitmapAdjustPrefetchIterator(BitmapHeapScanState *node); -static inline void BitmapAdjustPrefetchTarget(BitmapHeapScanState *node); -static inline void BitmapPrefetch(BitmapHeapScanState *node, - TableScanDesc scan); static bool BitmapShouldInitializeSharedState(ParallelBitmapHeapState *pstate); static BitmapHeapIterator *bhs_begin_iterate(TIDBitmap *tbm, dsa_pointer shared_area, @@ -122,7 +118,6 @@ BitmapHeapNext(BitmapHeapScanState *node) TableScanDesc scan; TIDBitmap *tbm; TupleTableSlot *slot; - ParallelBitmapHeapState *pstate = node->pstate; dsa_area *dsa = node->ss.ps.state->es_query_dsa; /* @@ -142,7 +137,7 @@ BitmapHeapNext(BitmapHeapScanState *node) * prefetching. node->prefetch_pages tracks exactly how many pages ahead * the prefetch iterator is. Also, node->prefetch_target tracks the * desired prefetch distance, which starts small and increases up to the - * node->prefetch_maximum. This is to avoid doing a lot of prefetching in + * scan->prefetch_maximum. This is to avoid doing a lot of prefetching in * a scan that stops after a few tuples because of a LIMIT. */ if (!node->initialized) @@ -154,7 +149,16 @@ BitmapHeapNext(BitmapHeapScanState *node) bool init_shared_state = node->pstate ? BitmapShouldInitializeSharedState(node->pstate) : false; - if (!pstate || init_shared_state) + /* + * Maximum number of prefetches for the tablespace if configured, + * otherwise the current value of the effective_io_concurrency GUC. + */ + int pf_maximum = 0; +#ifdef USE_PREFETCH + pf_maximum = get_tablespace_io_concurrency(node->ss.ss_currentRelation->rd_rel->reltablespace); +#endif + + if (!node->pstate || init_shared_state) { tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); @@ -169,23 +173,16 @@ BitmapHeapNext(BitmapHeapScanState *node) * dsa_pointer of the iterator state which will be used by * multiple processes to iterate jointly. */ - pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); + node->pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (pf_maximum > 0) { - pstate->prefetch_iterator = + node->pstate->prefetch_iterator = tbm_prepare_shared_iterate(tbm); - - /* - * We don't need the mutex here as we haven't yet woke up - * others. - */ - pstate->prefetch_pages = 0; - pstate->prefetch_target = -1; } #endif /* We have initialized the shared state so wake up others. */ - BitmapDoneInitializingSharedState(pstate); + BitmapDoneInitializingSharedState(node->pstate); } } @@ -216,19 +213,19 @@ BitmapHeapNext(BitmapHeapScanState *node) extra_flags); } + scan->prefetch_maximum = pf_maximum; + scan->bm_parallel = node->pstate; + scan->rs_bhs_iterator = bhs_begin_iterate(tbm, - pstate ? pstate->tbmiterator : InvalidDsaPointer, + scan->bm_parallel ? scan->bm_parallel->tbmiterator : InvalidDsaPointer, dsa); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (scan->prefetch_maximum > 0) { - node->pf_iterator = bhs_begin_iterate(tbm, - pstate ? pstate->prefetch_iterator : InvalidDsaPointer, - dsa); - /* Only used for serial BHS */ - node->prefetch_pages = 0; - node->prefetch_target = -1; + scan->rs_pf_bhs_iterator = bhs_begin_iterate(tbm, + scan->bm_parallel ? scan->bm_parallel->prefetch_iterator : InvalidDsaPointer, + dsa); } #endif /* USE_PREFETCH */ @@ -243,36 +240,6 @@ BitmapHeapNext(BitmapHeapScanState *node) { CHECK_FOR_INTERRUPTS(); -#ifdef USE_PREFETCH - - /* - * Try to prefetch at least a few pages even before we get to the - * second page if we don't stop reading after the first tuple. - */ - if (!pstate) - { - if (node->prefetch_target < node->prefetch_maximum) - node->prefetch_target++; - } - else if (pstate->prefetch_target < node->prefetch_maximum) - { - /* take spinlock while updating shared state */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target < node->prefetch_maximum) - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ - - /* - * We issue prefetch requests *after* fetching the current page to - * try to avoid having prefetching interfere with the main I/O. - * Also, this should happen only when we have determined there is - * still something to do on the current page, else we may - * uselessly prefetch the same page we are just about to request - * for real. - */ - BitmapPrefetch(node, scan); /* * If we are using lossy info, we have to recheck the qual @@ -296,23 +263,9 @@ BitmapHeapNext(BitmapHeapScanState *node) new_page: - BitmapAdjustPrefetchIterator(node); - - if (!table_scan_bitmap_next_block(scan, &node->recheck, &node->blockno, + if (!table_scan_bitmap_next_block(scan, &node->recheck, &scan->blockno, &node->lossy_pages, &node->exact_pages)) break; - - /* - * If serial, we can error out if the the prefetch block doesn't stay - * ahead of the current block. - */ - if (node->pstate == NULL && - node->pf_iterator && - node->pfblockno > node->blockno) - elog(ERROR, "prefetch and main iterators are out of sync"); - - /* Adjust the prefetch target */ - BitmapAdjustPrefetchTarget(node); } /* @@ -336,219 +289,6 @@ BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate) ConditionVariableBroadcast(&pstate->cv); } -/* - * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator - * - * We keep track of how far the prefetch iterator is ahead of the main - * iterator in prefetch_pages. For each block the main iterator returns, we - * decrement prefetch_pages. - */ -static inline void -BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - BitmapHeapIterator *prefetch_iterator = node->pf_iterator; - TBMIterateResult tbmpre; - - if (pstate == NULL) - { - if (node->prefetch_pages > 0) - { - /* The main iterator has closed the distance by one page */ - node->prefetch_pages--; - } - else if (prefetch_iterator) - { - /* Do not let the prefetch iterator get behind the main one */ - bhs_iterate(prefetch_iterator, &tbmpre); - node->pfblockno = tbmpre.blockno; - } - return; - } - - /* - * Adjusting the prefetch iterator before invoking - * table_scan_bitmap_next_block() keeps prefetch distance higher across - * the parallel workers. - */ - if (node->prefetch_maximum > 0) - { - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages > 0) - { - pstate->prefetch_pages--; - SpinLockRelease(&pstate->mutex); - } - else - { - /* Release the mutex before iterating */ - SpinLockRelease(&pstate->mutex); - - /* - * In case of shared mode, we can not ensure that the current - * blockno of the main iterator and that of the prefetch iterator - * are same. It's possible that whatever blockno we are - * prefetching will be processed by another process. Therefore, - * we don't validate the blockno here as we do in non-parallel - * case. - */ - if (prefetch_iterator) - { - bhs_iterate(prefetch_iterator, &tbmpre); - node->pfblockno = tbmpre.blockno; - } - } - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapAdjustPrefetchTarget - Adjust the prefetch target - * - * Increase prefetch target if it's not yet at the max. Note that - * we will increase it to zero after fetching the very first - * page/tuple, then to one after the second tuple is fetched, then - * it doubles as later pages are fetched. - */ -static inline void -BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - - if (pstate == NULL) - { - if (node->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (node->prefetch_target >= node->prefetch_maximum / 2) - node->prefetch_target = node->prefetch_maximum; - else if (node->prefetch_target > 0) - node->prefetch_target *= 2; - else - node->prefetch_target++; - return; - } - - /* Do an unlocked check first to save spinlock acquisitions. */ - if (pstate->prefetch_target < node->prefetch_maximum) - { - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (pstate->prefetch_target >= node->prefetch_maximum / 2) - pstate->prefetch_target = node->prefetch_maximum; - else if (pstate->prefetch_target > 0) - pstate->prefetch_target *= 2; - else - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target - */ -static inline void -BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - BitmapHeapIterator *prefetch_iterator = node->pf_iterator; - - if (pstate == NULL) - { - if (prefetch_iterator) - { - while (node->prefetch_pages < node->prefetch_target) - { - TBMIterateResult tbmpre; - bool skip_fetch; - - bhs_iterate(prefetch_iterator, &tbmpre); - - if (!BlockNumberIsValid(tbmpre.blockno)) - { - /* No more pages to prefetch */ - bhs_end_iterate(prefetch_iterator); - node->pf_iterator = NULL; - break; - } - node->prefetch_pages++; - node->pfblockno = tbmpre.blockno; - - /* - * If we expect not to have to actually read this heap page, - * skip this prefetch call, but continue to run the prefetch - * logic normally. (Would it be better not to increment - * prefetch_pages?) - */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre.recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre.blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre.blockno); - } - } - - return; - } - - if (pstate->prefetch_pages < pstate->prefetch_target) - { - if (prefetch_iterator) - { - while (1) - { - TBMIterateResult tbmpre; - bool do_prefetch = false; - bool skip_fetch; - - /* - * Recheck under the mutex. If some other process has already - * done enough prefetching then we need not to do anything. - */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages < pstate->prefetch_target) - { - pstate->prefetch_pages++; - do_prefetch = true; - } - SpinLockRelease(&pstate->mutex); - - if (!do_prefetch) - return; - - bhs_iterate(prefetch_iterator, &tbmpre); - if (!BlockNumberIsValid(tbmpre.blockno)) - { - /* No more pages to prefetch */ - bhs_end_iterate(prefetch_iterator); - node->pf_iterator = NULL; - break; - } - - node->pfblockno = tbmpre.blockno; - - /* As above, skip prefetch if we expect not to need page */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre.recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre.blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre.blockno); - } - } - } -#endif /* USE_PREFETCH */ -} - /* * BitmapHeapRecheck -- access method routine to recheck a tuple in EvalPlanQual */ @@ -594,22 +334,12 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) if (node->ss.ss_currentScanDesc) table_rescan(node->ss.ss_currentScanDesc, NULL); - /* release bitmaps and buffers if any */ - if (node->pf_iterator) - { - bhs_end_iterate(node->pf_iterator); - node->pf_iterator = NULL; - } + /* release bitmaps if any */ if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); node->tbm = NULL; node->initialized = false; - node->pvmbuffer = InvalidBuffer; node->recheck = true; - node->blockno = InvalidBlockNumber; - node->pfblockno = InvalidBlockNumber; ExecScanReScan(&node->ss); @@ -648,14 +378,10 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) table_endscan(scanDesc); /* - * release bitmaps and buffers if any + * release bitmaps if any */ - if (node->pf_iterator) - bhs_end_iterate(node->pf_iterator); if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); } /* ---------------------------------------------------------------- @@ -688,17 +414,11 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ps.ExecProcNode = ExecBitmapHeapScan; scanstate->tbm = NULL; - scanstate->pvmbuffer = InvalidBuffer; scanstate->exact_pages = 0; scanstate->lossy_pages = 0; - scanstate->pf_iterator = NULL; - scanstate->prefetch_pages = 0; - scanstate->prefetch_target = 0; scanstate->initialized = false; scanstate->pstate = NULL; scanstate->recheck = true; - scanstate->blockno = InvalidBlockNumber; - scanstate->pfblockno = InvalidBlockNumber; /* * Miscellaneous initialization @@ -738,13 +458,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->bitmapqualorig = ExecInitQual(node->bitmapqualorig, (PlanState *) scanstate); - /* - * Maximum number of prefetches for the tablespace if configured, - * otherwise the current value of the effective_io_concurrency GUC. - */ - scanstate->prefetch_maximum = - get_tablespace_io_concurrency(currentRelation->rd_rel->reltablespace); - scanstate->ss.ss_currentRelation = currentRelation; /* @@ -828,7 +541,7 @@ ExecBitmapHeapInitializeDSM(BitmapHeapScanState *node, /* Initialize the mutex */ SpinLockInit(&pstate->mutex); pstate->prefetch_pages = 0; - pstate->prefetch_target = 0; + pstate->prefetch_target = -1; pstate->state = BM_INITIAL; ConditionVariableInit(&pstate->cv); diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index cef54e2d5d..29fdd55893 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -82,6 +82,23 @@ typedef struct HeapScanDescData Buffer rs_vmbuffer; int rs_empty_tuples_pending; + /* + * These fields only used for prefetching in bitmap table scans + */ + + /* buffer for visibility-map lookups of prefetched pages */ + Buffer pvmbuffer; + + /* + * These fields only used in serial BHS + */ + /* Current target for prefetch distance */ + int prefetch_target; + /* # pages prefetch iterator is ahead of current */ + int prefetch_pages; + /* used to validate prefetch block stays ahead of current block */ + BlockNumber pfblockno; + /* these fields only used in page-at-a-time mode and for bitmap scans */ int rs_cindex; /* current tuple's index in vistuples */ int rs_ntuples; /* number of visible tuples on page */ diff --git a/src/include/access/relscan.h b/src/include/access/relscan.h index fb22f305bf..7938b741d6 100644 --- a/src/include/access/relscan.h +++ b/src/include/access/relscan.h @@ -26,6 +26,7 @@ struct ParallelTableScanDescData; struct BitmapHeapIterator; +struct ParallelBitmapHeapState; /* * Generic descriptor for table scans. This is the base-class for table scans, @@ -45,6 +46,13 @@ typedef struct TableScanDescData /* Only used for Bitmap table scans */ struct BitmapHeapIterator *rs_bhs_iterator; + struct BitmapHeapIterator *rs_pf_bhs_iterator; + + /* maximum value for prefetch_target */ + int prefetch_maximum; + struct ParallelBitmapHeapState *bm_parallel; + /* used to validate BHS prefetch and current block stay in sync */ + BlockNumber blockno; /* * Information about type and behaviour of the scan, a bitmask of members diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index edc54ecffe..284ea3d864 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -800,17 +800,6 @@ typedef struct TableAmRoutine * lossy_pages is incremented if the block's representation in the bitmap * is lossy, otherwise, exact_pages is incremented. * - * XXX: Currently this may only be implemented if the AM uses md.c as its - * storage manager, and uses ItemPointer->ip_blkid in a manner that maps - * blockids directly to the underlying storage. nodeBitmapHeapscan.c - * performs prefetching directly using that interface. This probably - * needs to be rectified at a later point. - * - * XXX: Currently this may only be implemented if the AM uses the - * visibilitymap, as nodeBitmapHeapscan.c unconditionally accesses it to - * perform prefetching. This probably needs to be rectified at a later - * point. - * * Optional callback, but either both scan_bitmap_next_block and * scan_bitmap_next_tuple need to exist, or neither. */ @@ -960,6 +949,9 @@ table_beginscan_bm(Relation rel, Snapshot snapshot, result = rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); result->rs_bhs_iterator = NULL; + result->rs_pf_bhs_iterator = NULL; + result->prefetch_maximum = 0; + result->bm_parallel = NULL; return result; } @@ -1024,6 +1016,12 @@ table_endscan(TableScanDesc scan) { bhs_end_iterate(scan->rs_bhs_iterator); scan->rs_bhs_iterator = NULL; + + if (scan->rs_pf_bhs_iterator) + { + bhs_end_iterate(scan->rs_pf_bhs_iterator); + scan->rs_pf_bhs_iterator = NULL; + } } scan->rs_rd->rd_tableam->scan_end(scan); @@ -1040,6 +1038,12 @@ table_rescan(TableScanDesc scan, { bhs_end_iterate(scan->rs_bhs_iterator); scan->rs_bhs_iterator = NULL; + + if (scan->rs_pf_bhs_iterator) + { + bhs_end_iterate(scan->rs_pf_bhs_iterator); + scan->rs_pf_bhs_iterator = NULL; + } } scan->rs_rd->rd_tableam->scan_rescan(scan, key, false, false, false, false); diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 52cedd1b35..60916bf0d0 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1785,18 +1785,11 @@ struct BitmapHeapIterator; * * bitmapqualorig execution state for bitmapqualorig expressions * tbm bitmap obtained from child index scan(s) - * pvmbuffer buffer for visibility-map lookups of prefetched pages * exact_pages total number of exact pages retrieved * lossy_pages total number of lossy pages retrieved - * pf_iterator for prefetching ahead of current page - * prefetch_pages # pages prefetch iterator is ahead of current - * prefetch_target current target prefetch distance - * prefetch_maximum maximum value for prefetch_target * initialized is node is ready to iterate * pstate shared state for parallel bitmap scan * recheck do current page's tuples need recheck - * blockno used to validate pf and current block in sync - * pfblockno used to validate pf stays ahead of current block * ---------------- */ typedef struct BitmapHeapScanState @@ -1804,18 +1797,11 @@ typedef struct BitmapHeapScanState ScanState ss; /* its first field is NodeTag */ ExprState *bitmapqualorig; TIDBitmap *tbm; - Buffer pvmbuffer; long exact_pages; long lossy_pages; - int prefetch_pages; - int prefetch_target; - int prefetch_maximum; bool initialized; - struct BitmapHeapIterator *pf_iterator; ParallelBitmapHeapState *pstate; bool recheck; - BlockNumber blockno; - BlockNumber pfblockno; } BitmapHeapScanState; /* ---------------- -- 2.40.1 --3o7pc6dfau5a5hry Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v10-0015-Remove-table_scan_bitmap_next_block.patch" ^ permalink raw reply [nested|flat] 10+ messages in thread
* [PATCH v9 13/17] Push BitmapHeapScan prefetch code into heapam.c @ 2024-03-22 13:42 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 10+ messages in thread From: Melanie Plageman @ 2024-03-22 13:42 UTC (permalink / raw) In preparation for transitioning to using the streaming read API for prefetching [1], move all of the BitmapHeapScanState members related to prefetching and the functions for accessing them into the HeapScanDescData and TableScanDescData. Members that still need to be accessed in BitmapHeapNext() could not be moved into heap AM-specific code. Specifically, parallel iterator setup requires several components which seem odd to pass to the table AM API. [1] https://www.postgresql.org/message-id/flat/CA%2BhUKGJkOiOCa%2Bmag4BF%2BzHo7qo%3Do9CFheB8%3Dg6uT5TUm2... --- src/backend/access/heap/heapam.c | 26 ++ src/backend/access/heap/heapam_handler.c | 268 +++++++++++++++ src/backend/executor/nodeBitmapHeapscan.c | 397 +++------------------- src/include/access/heapam.h | 12 + src/include/access/relscan.h | 11 + src/include/access/tableam.h | 38 ++- src/include/nodes/execnodes.h | 16 - 7 files changed, 388 insertions(+), 380 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index e7bed84f75..c12563a188 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -951,8 +951,16 @@ heap_beginscan(Relation relation, Snapshot snapshot, scan->rs_base.rs_flags = flags; scan->rs_base.rs_parallel = parallel_scan; scan->rs_strategy = NULL; /* set in initscan */ + + scan->rs_base.blockno = InvalidBlockNumber; + scan->rs_vmbuffer = InvalidBuffer; scan->rs_empty_tuples_pending = 0; + scan->pvmbuffer = InvalidBuffer; + + scan->pfblockno = InvalidBlockNumber; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; /* * Disable page-at-a-time mode if it's not a MVCC-safe snapshot. @@ -1035,6 +1043,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, scan->rs_base.rs_flags &= ~SO_ALLOW_PAGEMODE; } + scan->rs_base.blockno = InvalidBlockNumber; + + scan->pfblockno = InvalidBlockNumber; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; + /* * unpin scan buffers */ @@ -1047,6 +1061,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, scan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(scan->pvmbuffer)) + { + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + } + /* * reinitialize scan descriptor */ @@ -1072,6 +1092,12 @@ heap_endscan(TableScanDesc sscan) scan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(scan->pvmbuffer)) + { + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + } + /* * decrement relation reference count and free scan descriptor storage */ diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index adfc77684a..efd2784e03 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -55,6 +55,9 @@ static bool SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, OffsetNumber tupoffset); static BlockNumber heapam_scan_get_blocks_done(HeapScanDesc hscan); +static inline void BitmapAdjustPrefetchIterator(HeapScanDesc scan); +static inline void BitmapAdjustPrefetchTarget(HeapScanDesc scan); +static inline void BitmapPrefetch(HeapScanDesc scan); static const TableAmRoutine heapam_methods; @@ -2112,6 +2115,76 @@ heapam_estimate_rel_size(Relation rel, int32 *attr_widths, * ------------------------------------------------------------------------ */ +/* + * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator + * + * We keep track of how far the prefetch iterator is ahead of the main + * iterator in prefetch_pages. For each block the main iterator returns, we + * decrement prefetch_pages. + */ +static inline void +BitmapAdjustPrefetchIterator(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + TBMIterateResult tbmpre; + + if (pstate == NULL) + { + TBMIterator *prefetch_iterator = scan->rs_base.pf_tbmiterator; + + if (scan->prefetch_pages > 0) + { + /* The main iterator has closed the distance by one page */ + scan->prefetch_pages--; + } + else if (prefetch_iterator) + { + /* Do not let the prefetch iterator get behind the main one */ + tbm_iterate(prefetch_iterator, &tbmpre); + scan->pfblockno = tbmpre.blockno; + } + return; + } + + /* + * Adjusting the prefetch iterator before invoking + * table_scan_bitmap_next_block() keeps prefetch distance higher across + * the parallel workers. + */ + if (scan->rs_base.prefetch_maximum > 0) + { + TBMSharedIterator *prefetch_iterator = scan->rs_base.pf_shared_tbmiterator; + + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages > 0) + { + pstate->prefetch_pages--; + SpinLockRelease(&pstate->mutex); + } + else + { + /* Release the mutex before iterating */ + SpinLockRelease(&pstate->mutex); + + /* + * In case of shared mode, we can not ensure that the current + * blockno of the main iterator and that of the prefetch iterator + * are same. It's possible that whatever blockno we are + * prefetching will be processed by another process. Therefore, + * we don't validate the blockno here as we do in non-parallel + * case. + */ + if (prefetch_iterator) + { + tbm_shared_iterate(prefetch_iterator, &tbmpre); + scan->pfblockno = tbmpre.blockno; + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_block(TableScanDesc scan, bool *recheck, BlockNumber *blockno, @@ -2130,6 +2203,8 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, *blockno = InvalidBlockNumber; *recheck = true; + BitmapAdjustPrefetchIterator(hscan); + do { CHECK_FOR_INTERRUPTS(); @@ -2273,6 +2348,18 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, else (*exact_pages)++; + /* + * If serial, we can error out if the the prefetch block doesn't stay + * ahead of the current block. + */ + if (scan->bm_parallel == NULL && + scan->pf_tbmiterator && + hscan->pfblockno > hscan->rs_base.blockno) + elog(ERROR, "prefetch and main iterators are out of sync"); + + /* Adjust the prefetch target */ + BitmapAdjustPrefetchTarget(hscan); + /* * Return true to indicate that a valid block was found and the bitmap is * not exhausted. If there are no visible tuples on this page, @@ -2283,6 +2370,157 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, return true; } +/* + * BitmapAdjustPrefetchTarget - Adjust the prefetch target + * + * Increase prefetch target if it's not yet at the max. Note that + * we will increase it to zero after fetching the very first + * page/tuple, then to one after the second tuple is fetched, then + * it doubles as later pages are fetched. + */ +static inline void +BitmapAdjustPrefetchTarget(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + int prefetch_maximum = scan->rs_base.prefetch_maximum; + + if (pstate == NULL) + { + if (scan->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (scan->prefetch_target >= prefetch_maximum / 2) + scan->prefetch_target = prefetch_maximum; + else if (scan->prefetch_target > 0) + scan->prefetch_target *= 2; + else + scan->prefetch_target++; + return; + } + + /* Do an unlocked check first to save spinlock acquisitions. */ + if (pstate->prefetch_target < prefetch_maximum) + { + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_target >= prefetch_maximum) + /* don't increase any further */ ; + else if (pstate->prefetch_target >= prefetch_maximum / 2) + pstate->prefetch_target = prefetch_maximum; + else if (pstate->prefetch_target > 0) + pstate->prefetch_target *= 2; + else + pstate->prefetch_target++; + SpinLockRelease(&pstate->mutex); + } +#endif /* USE_PREFETCH */ +} + + +/* + * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target + */ +static inline void +BitmapPrefetch(HeapScanDesc scan) +{ +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->rs_base.bm_parallel; + + if (pstate == NULL) + { + TBMIterator *prefetch_iterator = scan->rs_base.pf_tbmiterator; + + if (prefetch_iterator) + { + while (scan->prefetch_pages < scan->prefetch_target) + { + TBMIterateResult tbmpre; + bool skip_fetch; + + tbm_iterate(prefetch_iterator, &tbmpre); + + if (!BlockNumberIsValid(tbmpre.blockno)) + { + /* No more pages to prefetch */ + tbm_end_iterate(prefetch_iterator); + scan->rs_base.pf_tbmiterator = NULL; + break; + } + scan->prefetch_pages++; + scan->pfblockno = tbmpre.blockno; + + /* + * If we expect not to have to actually read this heap page, + * skip this prefetch call, but continue to run the prefetch + * logic normally. (Would it be better not to increment + * prefetch_pages?) + */ + skip_fetch = (!(scan->rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre.recheck && + VM_ALL_VISIBLE(scan->rs_base.rs_rd, + tbmpre.blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(scan->rs_base.rs_rd, MAIN_FORKNUM, tbmpre.blockno); + } + } + + return; + } + + if (pstate->prefetch_pages < pstate->prefetch_target) + { + TBMSharedIterator *prefetch_iterator = scan->rs_base.pf_shared_tbmiterator; + + if (prefetch_iterator) + { + while (1) + { + TBMIterateResult tbmpre; + bool do_prefetch = false; + bool skip_fetch; + + /* + * Recheck under the mutex. If some other process has already + * done enough prefetching then we need not to do anything. + */ + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_pages < pstate->prefetch_target) + { + pstate->prefetch_pages++; + do_prefetch = true; + } + SpinLockRelease(&pstate->mutex); + + if (!do_prefetch) + return; + + tbm_shared_iterate(prefetch_iterator, &tbmpre); + if (!BlockNumberIsValid(tbmpre.blockno)) + { + /* No more pages to prefetch */ + tbm_end_shared_iterate(prefetch_iterator); + scan->rs_base.pf_shared_tbmiterator = NULL; + break; + } + + scan->pfblockno = tbmpre.blockno; + + /* As above, skip prefetch if we expect not to need page */ + skip_fetch = (!(scan->rs_base.rs_flags & SO_NEED_TUPLE) && + !tbmpre.recheck && + VM_ALL_VISIBLE(scan->rs_base.rs_rd, + tbmpre.blockno, + &scan->pvmbuffer)); + + if (!skip_fetch) + PrefetchBuffer(scan->rs_base.rs_rd, MAIN_FORKNUM, tbmpre.blockno); + } + } + } +#endif /* USE_PREFETCH */ +} + static bool heapam_scan_bitmap_next_tuple(TableScanDesc scan, TupleTableSlot *slot) @@ -2308,6 +2546,36 @@ heapam_scan_bitmap_next_tuple(TableScanDesc scan, if (hscan->rs_cindex < 0 || hscan->rs_cindex >= hscan->rs_ntuples) return false; +#ifdef USE_PREFETCH + + /* + * Try to prefetch at least a few pages even before we get to the second + * page if we don't stop reading after the first tuple. + */ + if (!scan->bm_parallel) + { + if (hscan->prefetch_target < scan->prefetch_maximum) + hscan->prefetch_target++; + } + else if (scan->bm_parallel->prefetch_target < scan->prefetch_maximum) + { + /* take spinlock while updating shared state */ + SpinLockAcquire(&scan->bm_parallel->mutex); + if (scan->bm_parallel->prefetch_target < scan->prefetch_maximum) + scan->bm_parallel->prefetch_target++; + SpinLockRelease(&scan->bm_parallel->mutex); + } + + /* + * We issue prefetch requests *after* fetching the current page to try to + * avoid having prefetching interfere with the main I/O. Also, this should + * happen only when we have determined there is still something to do on + * the current page, else we may uselessly prefetch the same page we are + * just about to request for real. + */ + BitmapPrefetch(hscan); +#endif /* USE_PREFETCH */ + targoffset = hscan->rs_vistuples[hscan->rs_cindex]; page = BufferGetPage(hscan->rs_cbuf); lp = PageGetItemId(page, targoffset); diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index 51c4360205..f241f4cb2c 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -51,10 +51,6 @@ static TupleTableSlot *BitmapHeapNext(BitmapHeapScanState *node); static inline void BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate); -static inline void BitmapAdjustPrefetchIterator(BitmapHeapScanState *node); -static inline void BitmapAdjustPrefetchTarget(BitmapHeapScanState *node); -static inline void BitmapPrefetch(BitmapHeapScanState *node, - TableScanDesc scan); static bool BitmapShouldInitializeSharedState(ParallelBitmapHeapState *pstate); @@ -71,7 +67,6 @@ BitmapHeapNext(BitmapHeapScanState *node) TableScanDesc scan; TIDBitmap *tbm; TupleTableSlot *slot; - ParallelBitmapHeapState *pstate = node->pstate; dsa_area *dsa = node->ss.ps.state->es_query_dsa; /* @@ -91,83 +86,53 @@ BitmapHeapNext(BitmapHeapScanState *node) * prefetching. node->prefetch_pages tracks exactly how many pages ahead * the prefetch iterator is. Also, node->prefetch_target tracks the * desired prefetch distance, which starts small and increases up to the - * node->prefetch_maximum. This is to avoid doing a lot of prefetching in + * scan->prefetch_maximum. This is to avoid doing a lot of prefetching in * a scan that stops after a few tuples because of a LIMIT. */ if (!node->initialized) { - TBMIterator *tbmiterator = NULL; - TBMSharedIterator *shared_tbmiterator = NULL; + /* + * The leader will immediately come out of the function, but others + * will be blocked until leader populates the TBM and wakes them up. + */ + bool init_shared_state = node->pstate ? + BitmapShouldInitializeSharedState(node->pstate) : false; + + /* + * Maximum number of prefetches for the tablespace if configured, + * otherwise the current value of the effective_io_concurrency GUC. + */ + int pf_maximum = 0; +#ifdef USE_PREFETCH + pf_maximum = get_tablespace_io_concurrency(node->ss.ss_currentRelation->rd_rel->reltablespace); +#endif - if (!pstate) + if (!node->pstate || init_shared_state) { tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); if (!tbm || !IsA(tbm, TIDBitmap)) elog(ERROR, "unrecognized result from subplan"); - node->tbm = tbm; - tbmiterator = tbm_begin_iterate(tbm); -#ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (init_shared_state) { - node->prefetch_iterator = tbm_begin_iterate(tbm); - node->prefetch_pages = 0; - node->prefetch_target = -1; - } -#endif /* USE_PREFETCH */ - } - else - { - /* - * The leader will immediately come out of the function, but - * others will be blocked until leader populates the TBM and wakes - * them up. - */ - if (BitmapShouldInitializeSharedState(pstate)) - { - tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); - if (!tbm || !IsA(tbm, TIDBitmap)) - elog(ERROR, "unrecognized result from subplan"); - - node->tbm = tbm; - /* * Prepare to iterate over the TBM. This will return the * dsa_pointer of the iterator state which will be used by * multiple processes to iterate jointly. */ - pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); + node->pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (pf_maximum > 0) { - pstate->prefetch_iterator = + node->pstate->prefetch_iterator = tbm_prepare_shared_iterate(tbm); - - /* - * We don't need the mutex here as we haven't yet woke up - * others. - */ - pstate->prefetch_pages = 0; - pstate->prefetch_target = -1; } #endif - /* We have initialized the shared state so wake up others. */ - BitmapDoneInitializingSharedState(pstate); - } - - /* Allocate a private iterator and attach the shared state to it */ - shared_tbmiterator = tbm_attach_shared_iterate(dsa, pstate->tbmiterator); - -#ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) - { - node->shared_prefetch_iterator = - tbm_attach_shared_iterate(dsa, pstate->prefetch_iterator); + BitmapDoneInitializingSharedState(node->pstate); } -#endif /* USE_PREFETCH */ } /* @@ -197,8 +162,26 @@ BitmapHeapNext(BitmapHeapScanState *node) extra_flags); } - scan->tbmiterator = tbmiterator; - scan->shared_tbmiterator = shared_tbmiterator; + scan->prefetch_maximum = pf_maximum; + scan->bm_parallel = node->pstate; + + if (!scan->bm_parallel) + scan->tbmiterator = tbm_begin_iterate(tbm); + else + /* Allocate a private iterator and attach the shared state to it */ + scan->shared_tbmiterator = tbm_attach_shared_iterate(dsa, scan->bm_parallel->tbmiterator); + +#ifdef USE_PREFETCH + if (scan->prefetch_maximum > 0) + { + if (!scan->bm_parallel) + scan->pf_tbmiterator = tbm_begin_iterate(tbm); + else + scan->pf_shared_tbmiterator = + tbm_attach_shared_iterate(dsa, scan->bm_parallel->prefetch_iterator); + } +#endif /* USE_PREFETCH */ + node->initialized = true; @@ -211,36 +194,6 @@ BitmapHeapNext(BitmapHeapScanState *node) { CHECK_FOR_INTERRUPTS(); -#ifdef USE_PREFETCH - - /* - * Try to prefetch at least a few pages even before we get to the - * second page if we don't stop reading after the first tuple. - */ - if (!pstate) - { - if (node->prefetch_target < node->prefetch_maximum) - node->prefetch_target++; - } - else if (pstate->prefetch_target < node->prefetch_maximum) - { - /* take spinlock while updating shared state */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target < node->prefetch_maximum) - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ - - /* - * We issue prefetch requests *after* fetching the current page to - * try to avoid having prefetching interfere with the main I/O. - * Also, this should happen only when we have determined there is - * still something to do on the current page, else we may - * uselessly prefetch the same page we are just about to request - * for real. - */ - BitmapPrefetch(node, scan); /* * If we are using lossy info, we have to recheck the qual @@ -264,23 +217,9 @@ BitmapHeapNext(BitmapHeapScanState *node) new_page: - BitmapAdjustPrefetchIterator(node); - - if (!table_scan_bitmap_next_block(scan, &node->recheck, &node->blockno, + if (!table_scan_bitmap_next_block(scan, &node->recheck, &scan->blockno, &node->lossy_pages, &node->exact_pages)) break; - - /* - * If serial, we can error out if the the prefetch block doesn't stay - * ahead of the current block. - */ - if (node->pstate == NULL && - node->prefetch_iterator && - node->pfblockno > node->blockno) - elog(ERROR, "prefetch and main iterators are out of sync"); - - /* Adjust the prefetch target */ - BitmapAdjustPrefetchTarget(node); } /* @@ -304,224 +243,7 @@ BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate) ConditionVariableBroadcast(&pstate->cv); } -/* - * BitmapAdjustPrefetchIterator - Adjust the prefetch iterator - * - * We keep track of how far the prefetch iterator is ahead of the main - * iterator in prefetch_pages. For each block the main iterator returns, we - * decrement prefetch_pages. - */ -static inline void -BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - TBMIterateResult tbmpre; - - if (pstate == NULL) - { - TBMIterator *prefetch_iterator = node->prefetch_iterator; - - if (node->prefetch_pages > 0) - { - /* The main iterator has closed the distance by one page */ - node->prefetch_pages--; - } - else if (prefetch_iterator) - { - /* Do not let the prefetch iterator get behind the main one */ - tbm_iterate(prefetch_iterator, &tbmpre); - node->pfblockno = tbmpre.blockno; - } - return; - } - - /* - * Adjusting the prefetch iterator before invoking - * table_scan_bitmap_next_block() keeps prefetch distance higher across - * the parallel workers. - */ - if (node->prefetch_maximum > 0) - { - TBMSharedIterator *prefetch_iterator = node->shared_prefetch_iterator; - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages > 0) - { - pstate->prefetch_pages--; - SpinLockRelease(&pstate->mutex); - } - else - { - /* Release the mutex before iterating */ - SpinLockRelease(&pstate->mutex); - - /* - * In case of shared mode, we can not ensure that the current - * blockno of the main iterator and that of the prefetch iterator - * are same. It's possible that whatever blockno we are - * prefetching will be processed by another process. Therefore, - * we don't validate the blockno here as we do in non-parallel - * case. - */ - if (prefetch_iterator) - { - tbm_shared_iterate(prefetch_iterator, &tbmpre); - node->pfblockno = tbmpre.blockno; - } - } - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapAdjustPrefetchTarget - Adjust the prefetch target - * - * Increase prefetch target if it's not yet at the max. Note that - * we will increase it to zero after fetching the very first - * page/tuple, then to one after the second tuple is fetched, then - * it doubles as later pages are fetched. - */ -static inline void -BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - - if (pstate == NULL) - { - if (node->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (node->prefetch_target >= node->prefetch_maximum / 2) - node->prefetch_target = node->prefetch_maximum; - else if (node->prefetch_target > 0) - node->prefetch_target *= 2; - else - node->prefetch_target++; - return; - } - - /* Do an unlocked check first to save spinlock acquisitions. */ - if (pstate->prefetch_target < node->prefetch_maximum) - { - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target >= node->prefetch_maximum) - /* don't increase any further */ ; - else if (pstate->prefetch_target >= node->prefetch_maximum / 2) - pstate->prefetch_target = node->prefetch_maximum; - else if (pstate->prefetch_target > 0) - pstate->prefetch_target *= 2; - else - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ -} - -/* - * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target - */ -static inline void -BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) -{ -#ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - - if (pstate == NULL) - { - TBMIterator *prefetch_iterator = node->prefetch_iterator; - - if (prefetch_iterator) - { - while (node->prefetch_pages < node->prefetch_target) - { - TBMIterateResult tbmpre; - bool skip_fetch; - - tbm_iterate(prefetch_iterator, &tbmpre); - - if (!BlockNumberIsValid(tbmpre.blockno)) - { - /* No more pages to prefetch */ - tbm_end_iterate(prefetch_iterator); - node->prefetch_iterator = NULL; - break; - } - node->prefetch_pages++; - node->pfblockno = tbmpre.blockno; - - /* - * If we expect not to have to actually read this heap page, - * skip this prefetch call, but continue to run the prefetch - * logic normally. (Would it be better not to increment - * prefetch_pages?) - */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre.recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre.blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre.blockno); - } - } - - return; - } - - if (pstate->prefetch_pages < pstate->prefetch_target) - { - TBMSharedIterator *prefetch_iterator = node->shared_prefetch_iterator; - - if (prefetch_iterator) - { - while (1) - { - TBMIterateResult tbmpre; - bool do_prefetch = false; - bool skip_fetch; - - /* - * Recheck under the mutex. If some other process has already - * done enough prefetching then we need not to do anything. - */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_pages < pstate->prefetch_target) - { - pstate->prefetch_pages++; - do_prefetch = true; - } - SpinLockRelease(&pstate->mutex); - - if (!do_prefetch) - return; - - tbm_shared_iterate(prefetch_iterator, &tbmpre); - if (!BlockNumberIsValid(tbmpre.blockno)) - { - /* No more pages to prefetch */ - tbm_end_shared_iterate(prefetch_iterator); - node->shared_prefetch_iterator = NULL; - break; - } - - node->pfblockno = tbmpre.blockno; - - /* As above, skip prefetch if we expect not to need page */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && - !tbmpre.recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, - tbmpre.blockno, - &node->pvmbuffer)); - - if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre.blockno); - } - } - } -#endif /* USE_PREFETCH */ -} /* * BitmapHeapRecheck -- access method routine to recheck a tuple in EvalPlanQual @@ -569,22 +291,11 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ - if (node->prefetch_iterator) - tbm_end_iterate(node->prefetch_iterator); - if (node->shared_prefetch_iterator) - tbm_end_shared_iterate(node->shared_prefetch_iterator); if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); node->tbm = NULL; - node->prefetch_iterator = NULL; node->initialized = false; - node->shared_prefetch_iterator = NULL; - node->pvmbuffer = InvalidBuffer; node->recheck = true; - node->blockno = InvalidBlockNumber; - node->pfblockno = InvalidBlockNumber; ExecScanReScan(&node->ss); @@ -625,14 +336,8 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * release bitmaps and buffers if any */ - if (node->prefetch_iterator) - tbm_end_iterate(node->prefetch_iterator); if (node->tbm) tbm_free(node->tbm); - if (node->shared_prefetch_iterator) - tbm_end_shared_iterate(node->shared_prefetch_iterator); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); } /* ---------------------------------------------------------------- @@ -665,18 +370,11 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ps.ExecProcNode = ExecBitmapHeapScan; scanstate->tbm = NULL; - scanstate->pvmbuffer = InvalidBuffer; scanstate->exact_pages = 0; scanstate->lossy_pages = 0; - scanstate->prefetch_iterator = NULL; - scanstate->prefetch_pages = 0; - scanstate->prefetch_target = 0; scanstate->initialized = false; - scanstate->shared_prefetch_iterator = NULL; scanstate->pstate = NULL; scanstate->recheck = true; - scanstate->blockno = InvalidBlockNumber; - scanstate->pfblockno = InvalidBlockNumber; /* * Miscellaneous initialization @@ -716,13 +414,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->bitmapqualorig = ExecInitQual(node->bitmapqualorig, (PlanState *) scanstate); - /* - * Maximum number of prefetches for the tablespace if configured, - * otherwise the current value of the effective_io_concurrency GUC. - */ - scanstate->prefetch_maximum = - get_tablespace_io_concurrency(currentRelation->rd_rel->reltablespace); - scanstate->ss.ss_currentRelation = currentRelation; /* @@ -806,7 +497,7 @@ ExecBitmapHeapInitializeDSM(BitmapHeapScanState *node, /* Initialize the mutex */ SpinLockInit(&pstate->mutex); pstate->prefetch_pages = 0; - pstate->prefetch_target = 0; + pstate->prefetch_target = -1; pstate->state = BM_INITIAL; ConditionVariableInit(&pstate->cv); diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 3dfb19ec7d..22bdccc2a9 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -81,6 +81,18 @@ typedef struct HeapScanDescData */ Buffer rs_vmbuffer; int rs_empty_tuples_pending; + /* buffer for visibility-map lookups of prefetched pages */ + Buffer pvmbuffer; + + /* + * These fields only used for prefetching in bitmap table scans + */ + /* Current target for prefetch distance */ + int prefetch_target; + /* # pages prefetch iterator is ahead of current */ + int prefetch_pages; + /* used to validate prefetch block stays ahead of current block */ + BlockNumber pfblockno; /* these fields only used in page-at-a-time mode and for bitmap scans */ int rs_cindex; /* current tuple's index in vistuples */ diff --git a/src/include/access/relscan.h b/src/include/access/relscan.h index 92b829cebc..93168bd350 100644 --- a/src/include/access/relscan.h +++ b/src/include/access/relscan.h @@ -26,6 +26,7 @@ struct ParallelTableScanDescData; struct TBMIterator; struct TBMSharedIterator; +struct ParallelBitmapHeapState; /* * Generic descriptor for table scans. This is the base-class for table scans, @@ -46,6 +47,16 @@ typedef struct TableScanDescData /* Only used for Bitmap table scans */ struct TBMIterator *tbmiterator; struct TBMSharedIterator *shared_tbmiterator; + /* Prefetch iterators */ + struct TBMIterator *pf_tbmiterator; + struct TBMSharedIterator *pf_shared_tbmiterator; + + /* maximum value for prefetch_target */ + int prefetch_maximum; + struct ParallelBitmapHeapState *bm_parallel; + + /* used to validate prefetch and current block stay in sync */ + BlockNumber blockno; /* * Information about type and behaviour of the scan, a bitmask of members diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 1d4b79a73f..9cab4462d6 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -800,17 +800,6 @@ typedef struct TableAmRoutine * lossy_pages is incremented if the block's representation in the bitmap * is lossy, otherwise, exact_pages is incremented. * - * XXX: Currently this may only be implemented if the AM uses md.c as its - * storage manager, and uses ItemPointer->ip_blkid in a manner that maps - * blockids directly to the underlying storage. nodeBitmapHeapscan.c - * performs prefetching directly using that interface. This probably - * needs to be rectified at a later point. - * - * XXX: Currently this may only be implemented if the AM uses the - * visibilitymap, as nodeBitmapHeapscan.c unconditionally accesses it to - * perform prefetching. This probably needs to be rectified at a later - * point. - * * Optional callback, but either both scan_bitmap_next_block and * scan_bitmap_next_tuple need to exist, or neither. */ @@ -961,6 +950,9 @@ table_beginscan_bm(Relation rel, Snapshot snapshot, result = rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); result->shared_tbmiterator = NULL; result->tbmiterator = NULL; + result->pf_shared_tbmiterator = NULL; + result->pf_tbmiterator = NULL; + result->bm_parallel = NULL; return result; } @@ -1029,11 +1021,23 @@ table_endscan(TableScanDesc scan) scan->shared_tbmiterator = NULL; } + if (scan->pf_shared_tbmiterator) + { + tbm_end_shared_iterate(scan->pf_shared_tbmiterator); + scan->pf_shared_tbmiterator = NULL; + } + if (scan->tbmiterator) { tbm_end_iterate(scan->tbmiterator); scan->tbmiterator = NULL; } + + if (scan->pf_tbmiterator) + { + tbm_end_iterate(scan->pf_tbmiterator); + scan->pf_tbmiterator = NULL; + } } scan->rs_rd->rd_tableam->scan_end(scan); @@ -1054,11 +1058,23 @@ table_rescan(TableScanDesc scan, scan->shared_tbmiterator = NULL; } + if (scan->pf_shared_tbmiterator) + { + tbm_end_shared_iterate(scan->pf_shared_tbmiterator); + scan->pf_shared_tbmiterator = NULL; + } + if (scan->tbmiterator) { tbm_end_iterate(scan->tbmiterator); scan->tbmiterator = NULL; } + + if (scan->pf_tbmiterator) + { + tbm_end_iterate(scan->pf_tbmiterator); + scan->pf_tbmiterator = NULL; + } } scan->rs_rd->rd_tableam->scan_rescan(scan, key, false, false, false, false); diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 8688bc5ab0..7a3fdf9cd4 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1783,19 +1783,11 @@ typedef struct ParallelBitmapHeapState * * bitmapqualorig execution state for bitmapqualorig expressions * tbm bitmap obtained from child index scan(s) - * pvmbuffer buffer for visibility-map lookups of prefetched pages * exact_pages total number of exact pages retrieved * lossy_pages total number of lossy pages retrieved - * prefetch_iterator iterator for prefetching ahead of current page - * prefetch_pages # pages prefetch iterator is ahead of current - * prefetch_target current target prefetch distance - * prefetch_maximum maximum value for prefetch_target * initialized is node is ready to iterate - * shared_prefetch_iterator shared iterator for prefetching * pstate shared state for parallel bitmap scan * recheck do current page's tuples need recheck - * blockno used to validate pf and current block in sync - * pfblockno used to validate pf stays ahead of current block * ---------------- */ typedef struct BitmapHeapScanState @@ -1803,19 +1795,11 @@ typedef struct BitmapHeapScanState ScanState ss; /* its first field is NodeTag */ ExprState *bitmapqualorig; TIDBitmap *tbm; - Buffer pvmbuffer; long exact_pages; long lossy_pages; - TBMIterator *prefetch_iterator; - int prefetch_pages; - int prefetch_target; - int prefetch_maximum; bool initialized; - TBMSharedIterator *shared_prefetch_iterator; ParallelBitmapHeapState *pstate; bool recheck; - BlockNumber blockno; - BlockNumber pfblockno; } BitmapHeapScanState; /* ---------------- -- 2.40.1 --7mdtsjmrzitrgzgx Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0014-Unify-parallel-and-serial-BitmapHeapScan-iterator.patch" ^ permalink raw reply [nested|flat] 10+ messages in thread
* [PATCH v16 16/18] Push BitmapHeapScan prefetch code into heap AM @ 2024-04-05 22:48 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 10+ messages in thread From: Melanie Plageman @ 2024-04-05 22:48 UTC (permalink / raw) An earlier commit eliminated a table AM layering violation for the current block in a BitmapHeapScan but left the violation in BitmapHeapScan prefetching code. To resolve this, move and manage all state used for prefetching entirely into the heap AM. Author: Melanie Plageman Discussion: https://postgr.es/m/CAAKRu_ZwCwWFeL_H3ia26bP2e7HiKLWt0ZmGXPVwPO6uXq0vaA%40mail.gmail.com --- src/backend/access/heap/heapam.c | 33 +++++- src/backend/access/heap/heapam_handler.c | 125 +++++++++++++++------- src/backend/executor/nodeBitmapHeapscan.c | 98 +++-------------- src/include/access/heapam.h | 29 ++++- src/include/access/tableam.h | 18 +--- src/include/executor/nodeBitmapHeapscan.h | 7 ++ src/include/nodes/execnodes.h | 19 ---- 7 files changed, 168 insertions(+), 161 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index c35dc3c4e7..5728b2ea8a 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -975,8 +975,10 @@ heap_beginscan_bm(Relation relation, Snapshot snapshot, uint32 flags) scan->iterator.serial = NULL; scan->iterator.parallel = NULL; - scan->vmbuffer = InvalidBuffer; + scan->pf_iterator.serial = NULL; + scan->pf_iterator.parallel = NULL; + scan->pvmbuffer = InvalidBuffer; return (TableScanDesc) scan; } @@ -1081,6 +1083,10 @@ heap_endscan_bm(TableScanDesc sscan) if (BufferIsValid(scan->vmbuffer)) ReleaseBuffer(scan->vmbuffer); + if (BufferIsValid(scan->pvmbuffer)) + ReleaseBuffer(scan->pvmbuffer); + + bhs_end_iterate(&scan->pf_iterator); bhs_end_iterate(&scan->iterator); /* @@ -1390,7 +1396,7 @@ heap_getnextslot_tidrange(TableScanDesc sscan, ScanDirection direction, void heap_rescan_bm(TableScanDesc sscan, TIDBitmap *tbm, - ParallelBitmapHeapState *pstate, dsa_area *dsa) + ParallelBitmapHeapState *pstate, dsa_area *dsa, int pf_maximum) { BitmapHeapScanDesc scan = (BitmapHeapScanDesc) sscan; @@ -1401,9 +1407,19 @@ heap_rescan_bm(TableScanDesc sscan, TIDBitmap *tbm, ReleaseBuffer(scan->vmbuffer); scan->vmbuffer = InvalidBuffer; + if (BufferIsValid(scan->pvmbuffer)) + ReleaseBuffer(scan->pvmbuffer); + scan->pvmbuffer = InvalidBuffer; + + bhs_end_iterate(&scan->pf_iterator); bhs_end_iterate(&scan->iterator); + scan->prefetch_maximum = 0; scan->empty_tuples_pending = 0; + scan->pstate = NULL; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; + scan->pfblockno = InvalidBlockNumber; /* * reinitialize heap scan descriptor @@ -1414,6 +1430,19 @@ heap_rescan_bm(TableScanDesc sscan, TIDBitmap *tbm, pstate ? pstate->tbmiterator : InvalidDsaPointer); + +#ifdef USE_PREFETCH + if (pf_maximum > 0) + { + bhs_begin_iterate(&scan->pf_iterator, tbm, dsa, + pstate ? + pstate->prefetch_iterator : + InvalidDsaPointer); + } +#endif /* USE_PREFETCH */ + + scan->pstate = pstate; + scan->prefetch_maximum = pf_maximum; } diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index ebaba33406..fca9c7233e 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -60,6 +60,9 @@ static bool SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, OffsetNumber tupoffset); static BlockNumber heapam_scan_get_blocks_done(HeapScanDesc hscan); +static inline void BitmapAdjustPrefetchIterator(BitmapHeapScanDesc scan); +static inline void BitmapAdjustPrefetchTarget(BitmapHeapScanDesc scan); +static inline void BitmapPrefetch(BitmapHeapScanDesc scan); static const TableAmRoutine heapam_methods; @@ -2187,26 +2190,26 @@ heapam_estimate_rel_size(Relation rel, int32 *attr_widths, * iterator in prefetch_pages. For each block the main iterator returns, we * decrement prefetch_pages. */ -void -BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) +static inline void +BitmapAdjustPrefetchIterator(BitmapHeapScanDesc scan) { #ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - BitmapHeapIterator *prefetch_iterator = &node->pf_iterator; + BitmapHeapIterator *prefetch_iterator = &scan->pf_iterator; + ParallelBitmapHeapState *pstate = scan->pstate; TBMIterateResult *tbmpre; if (pstate == NULL) { - if (node->prefetch_pages > 0) + if (scan->prefetch_pages > 0) { /* The main iterator has closed the distance by one page */ - node->prefetch_pages--; + scan->prefetch_pages--; } else if (!prefetch_iterator->exhausted) { /* Do not let the prefetch iterator get behind the main one */ tbmpre = bhs_iterate(prefetch_iterator); - node->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; + scan->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; } return; } @@ -2216,7 +2219,7 @@ BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) * table_scan_bitmap_next_block() keeps prefetch distance higher across * the parallel workers. */ - if (node->prefetch_maximum > 0) + if (scan->prefetch_maximum > 0) { SpinLockAcquire(&pstate->mutex); if (pstate->prefetch_pages > 0) @@ -2240,7 +2243,7 @@ BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) if (!prefetch_iterator->exhausted) { tbmpre = bhs_iterate(prefetch_iterator); - node->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; + scan->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; } } } @@ -2256,33 +2259,34 @@ BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) * page/tuple, then to one after the second tuple is fetched, then * it doubles as later pages are fetched. */ -void -BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) +static inline void +BitmapAdjustPrefetchTarget(BitmapHeapScanDesc scan) { #ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; + ParallelBitmapHeapState *pstate = scan->pstate; + int prefetch_maximum = scan->prefetch_maximum; if (pstate == NULL) { - if (node->prefetch_target >= node->prefetch_maximum) + if (scan->prefetch_target >= prefetch_maximum) /* don't increase any further */ ; - else if (node->prefetch_target >= node->prefetch_maximum / 2) - node->prefetch_target = node->prefetch_maximum; - else if (node->prefetch_target > 0) - node->prefetch_target *= 2; + else if (scan->prefetch_target >= prefetch_maximum / 2) + scan->prefetch_target = prefetch_maximum; + else if (scan->prefetch_target > 0) + scan->prefetch_target *= 2; else - node->prefetch_target++; + scan->prefetch_target++; return; } /* Do an unlocked check first to save spinlock acquisitions. */ - if (pstate->prefetch_target < node->prefetch_maximum) + if (pstate->prefetch_target < prefetch_maximum) { SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target >= node->prefetch_maximum) + if (pstate->prefetch_target >= prefetch_maximum) /* don't increase any further */ ; - else if (pstate->prefetch_target >= node->prefetch_maximum / 2) - pstate->prefetch_target = node->prefetch_maximum; + else if (pstate->prefetch_target >= prefetch_maximum / 2) + pstate->prefetch_target = prefetch_maximum; else if (pstate->prefetch_target > 0) pstate->prefetch_target *= 2; else @@ -2295,18 +2299,19 @@ BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) /* * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target */ -void -BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) +static inline void +BitmapPrefetch(BitmapHeapScanDesc scan) { #ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - BitmapHeapIterator *prefetch_iterator = &node->pf_iterator; + ParallelBitmapHeapState *pstate = scan->pstate; + BitmapHeapIterator *prefetch_iterator = &scan->pf_iterator; + Relation rel = scan->heap_common.rs_base.rs_rd; if (pstate == NULL) { if (!prefetch_iterator->exhausted) { - while (node->prefetch_pages < node->prefetch_target) + while (scan->prefetch_pages < scan->prefetch_target) { TBMIterateResult *tbmpre = bhs_iterate(prefetch_iterator); bool skip_fetch; @@ -2317,8 +2322,8 @@ BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) bhs_end_iterate(prefetch_iterator); break; } - node->prefetch_pages++; - node->pfblockno = tbmpre->blockno; + scan->prefetch_pages++; + scan->pfblockno = tbmpre->blockno; /* * If we expect not to have to actually read this heap page, @@ -2326,14 +2331,14 @@ BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) * logic normally. (Would it be better not to increment * prefetch_pages?) */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && + skip_fetch = (!(scan->heap_common.rs_base.rs_flags & SO_NEED_TUPLE) && !tbmpre->recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, + VM_ALL_VISIBLE(rel, tbmpre->blockno, - &node->pvmbuffer)); + &scan->pvmbuffer)); if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre->blockno); + PrefetchBuffer(rel, MAIN_FORKNUM, tbmpre->blockno); } } @@ -2373,17 +2378,17 @@ BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) break; } - node->pfblockno = tbmpre->blockno; + scan->pfblockno = tbmpre->blockno; /* As above, skip prefetch if we expect not to need page */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLE) && + skip_fetch = (!(scan->heap_common.rs_base.rs_flags & SO_NEED_TUPLE) && !tbmpre->recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, + VM_ALL_VISIBLE(rel, tbmpre->blockno, - &node->pvmbuffer)); + &scan->pvmbuffer)); if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre->blockno); + PrefetchBuffer(rel, MAIN_FORKNUM, tbmpre->blockno); } } } @@ -2416,6 +2421,8 @@ heapam_scan_bitmap_next_block(TableScanDesc sscan, *blockno = InvalidBlockNumber; *recheck = true; + BitmapAdjustPrefetchIterator(scan); + do { CHECK_FOR_INTERRUPTS(); @@ -2556,6 +2563,18 @@ heapam_scan_bitmap_next_block(TableScanDesc sscan, else (*exact_pages)++; + /* + * If serial, we can error out if the the prefetch block doesn't stay + * ahead of the current block. + */ + if (scan->pstate == NULL && + !scan->pf_iterator.exhausted && + scan->pfblockno < block) + elog(ERROR, "prefetch and main iterators are out of sync"); + + /* Adjust the prefetch target */ + BitmapAdjustPrefetchTarget(scan); + /* * Return true to indicate that a valid block was found and the bitmap is * not exhausted. If there are no visible tuples on this page, @@ -2592,6 +2611,36 @@ heapam_scan_bitmap_next_tuple(TableScanDesc scan, if (hscan->rs_cindex < 0 || hscan->rs_cindex >= hscan->rs_ntuples) return false; +#ifdef USE_PREFETCH + + /* + * Try to prefetch at least a few pages even before we get to the second + * page if we don't stop reading after the first tuple. + */ + if (!bscan->pstate) + { + if (bscan->prefetch_target < bscan->prefetch_maximum) + bscan->prefetch_target++; + } + else if (bscan->pstate->prefetch_target < bscan->prefetch_maximum) + { + /* take spinlock while updating shared state */ + SpinLockAcquire(&bscan->pstate->mutex); + if (bscan->pstate->prefetch_target < bscan->prefetch_maximum) + bscan->pstate->prefetch_target++; + SpinLockRelease(&bscan->pstate->mutex); + } + + /* + * We issue prefetch requests *after* fetching the current page to try to + * avoid having prefetching interfere with the main I/O. Also, this should + * happen only when we have determined there is still something to do on + * the current page, else we may uselessly prefetch the same page we are + * just about to request for real. + */ + BitmapPrefetch(bscan); +#endif /* USE_PREFETCH */ + targoffset = hscan->rs_vistuples[hscan->rs_cindex]; page = BufferGetPage(hscan->rs_cbuf); lp = PageGetItemId(page, targoffset); diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index 6f1afd427d..ae36dfd773 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -124,7 +124,6 @@ BitmapHeapNext(BitmapHeapScanState *node) TIDBitmap *tbm; TupleTableSlot *slot; ParallelBitmapHeapState *pstate = node->pstate; - dsa_area *dsa = node->ss.ps.state->es_query_dsa; /* * extract necessary information from index scan node @@ -148,7 +147,9 @@ BitmapHeapNext(BitmapHeapScanState *node) */ if (!node->initialized) { + Relation rel = node->ss.ss_currentRelation; uint32 extra_flags = 0; + bool pf_maximum = 0; /* * The leader will immediately come out of the function, but others @@ -157,6 +158,15 @@ BitmapHeapNext(BitmapHeapScanState *node) bool init_shared_state = node->pstate ? BitmapShouldInitializeSharedState(node->pstate) : false; + /* + * Maximum number of prefetches for the tablespace if configured, + * otherwise the current value of the effective_io_concurrency GUC. + */ +#ifdef USE_PREFETCH + pf_maximum = get_tablespace_io_concurrency(rel->rd_rel->reltablespace); +#endif + + if (!pstate || init_shared_state) { tbm = (TIDBitmap *) MultiExecProcNode(outerPlanState(node)); @@ -174,7 +184,7 @@ BitmapHeapNext(BitmapHeapScanState *node) */ pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (pf_maximum > 0) { pstate->prefetch_iterator = tbm_prepare_shared_iterate(tbm); @@ -200,25 +210,16 @@ BitmapHeapNext(BitmapHeapScanState *node) * scan descriptor and begin the scan. */ scan = table_beginscan_bm(node->ss.ss_currentScanDesc, - node->ss.ss_currentRelation, + rel, node->ss.ps.state->es_snapshot, extra_flags, + pf_maximum, node->tbm, node->pstate, node->ss.ps.state->es_query_dsa); node->ss.ss_currentScanDesc = scan; -#ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) - { - bhs_begin_iterate(&node->pf_iterator, tbm, dsa, - pstate ? - pstate->prefetch_iterator : - InvalidDsaPointer); - } -#endif /* USE_PREFETCH */ - node->initialized = true; goto new_page; @@ -230,37 +231,6 @@ BitmapHeapNext(BitmapHeapScanState *node) { CHECK_FOR_INTERRUPTS(); -#ifdef USE_PREFETCH - - /* - * Try to prefetch at least a few pages even before we get to the - * second page if we don't stop reading after the first tuple. - */ - if (!pstate) - { - if (node->prefetch_target < node->prefetch_maximum) - node->prefetch_target++; - } - else if (pstate->prefetch_target < node->prefetch_maximum) - { - /* take spinlock while updating shared state */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target < node->prefetch_maximum) - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ - - /* - * We issue prefetch requests *after* fetching the current page to - * try to avoid having prefetching interfere with the main I/O. - * Also, this should happen only when we have determined there is - * still something to do on the current page, else we may - * uselessly prefetch the same page we are just about to request - * for real. - */ - BitmapPrefetch(node, scan); - /* * If we are using lossy info, we have to recheck the qual * conditions at every tuple. @@ -283,23 +253,9 @@ BitmapHeapNext(BitmapHeapScanState *node) new_page: - BitmapAdjustPrefetchIterator(node); - if (!table_scan_bitmap_next_block(scan, &node->blockno, &node->recheck, &node->lossy_pages, &node->exact_pages)) break; - - /* - * If serial, we can error out if the the prefetch block doesn't stay - * ahead of the current block. - */ - if (node->pstate == NULL && - !node->pf_iterator.exhausted && - node->pfblockno < node->blockno) - elog(ERROR, "prefetch and main iterators are out of sync"); - - /* Adjust the prefetch target */ - BitmapAdjustPrefetchTarget(node); } /* @@ -365,21 +321,12 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* release bitmaps and buffers if any */ - if (node->pf_iterator.exhausted) - bhs_end_iterate(&node->pf_iterator); if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); node->tbm = NULL; node->initialized = false; - node->pvmbuffer = InvalidBuffer; node->recheck = true; node->blockno = InvalidBlockNumber; - node->pfblockno = InvalidBlockNumber; - /* Only used for serial BHS */ - node->prefetch_pages = 0; - node->prefetch_target = -1; ExecScanReScan(&node->ss); @@ -418,14 +365,10 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) table_endscan_bm(scanDesc); /* - * release bitmaps and buffers if any + * release bitmaps if any */ - if (!node->pf_iterator.exhausted) - bhs_end_iterate(&node->pf_iterator); if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); } /* ---------------------------------------------------------------- @@ -458,16 +401,12 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ps.ExecProcNode = ExecBitmapHeapScan; scanstate->tbm = NULL; - scanstate->pvmbuffer = InvalidBuffer; scanstate->exact_pages = 0; scanstate->lossy_pages = 0; - scanstate->prefetch_pages = 0; - scanstate->prefetch_target = -1; scanstate->initialized = false; scanstate->pstate = NULL; scanstate->recheck = true; scanstate->blockno = InvalidBlockNumber; - scanstate->pfblockno = InvalidBlockNumber; /* * Miscellaneous initialization @@ -507,13 +446,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->bitmapqualorig = ExecInitQual(node->bitmapqualorig, (PlanState *) scanstate); - /* - * Maximum number of prefetches for the tablespace if configured, - * otherwise the current value of the effective_io_concurrency GUC. - */ - scanstate->prefetch_maximum = - get_tablespace_io_concurrency(currentRelation->rd_rel->reltablespace); - scanstate->ss.ss_currentRelation = currentRelation; /* diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index f81a295685..499e739da7 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -93,6 +93,10 @@ typedef struct BitmapHeapScanDescData * Members common to Parallel and Serial BitmapHeapScan */ BitmapHeapIterator iterator; + BitmapHeapIterator pf_iterator; + + /* maximum value for prefetch_target */ + int prefetch_maximum; /* * These fields are only used for bitmap scans for the "skip fetch" @@ -102,7 +106,26 @@ typedef struct BitmapHeapScanDescData * to return. They are common to parallel and serial BitmapHeapScans */ Buffer vmbuffer; + /* buffer for visibility-map lookups of prefetched pages */ + Buffer pvmbuffer; int empty_tuples_pending; + + /* + * Parallel-only members + */ + + struct ParallelBitmapHeapState *pstate; + + /* + * Serial-only members + */ + + /* Current target for prefetch distance */ + int prefetch_target; + /* # pages prefetch iterator is ahead of current */ + int prefetch_pages; + /* used to validate prefetch block stays ahead of current block */ + BlockNumber pfblockno; } BitmapHeapScanDescData; typedef struct BitmapHeapScanDescData *BitmapHeapScanDesc; @@ -307,7 +330,7 @@ extern TableScanDesc heap_beginscan_bm(Relation relation, Snapshot snapshot, uin extern void heap_endscan_bm(TableScanDesc scan); extern void heap_rescan_bm(TableScanDesc sscan, TIDBitmap *tbm, - ParallelBitmapHeapState *pstate, dsa_area *dsa); + ParallelBitmapHeapState *pstate, dsa_area *dsa, int pf_maximum); extern bool heap_fetch(Relation relation, Snapshot snapshot, HeapTuple tuple, Buffer *userbuf, bool keep_buf); @@ -426,10 +449,6 @@ extern bool heapam_scan_analyze_next_tuple(TableScanDesc scan, double *liverows, double *deadrows, TupleTableSlot *slot); -extern void BitmapAdjustPrefetchIterator(BitmapHeapScanState *node); -extern void BitmapAdjustPrefetchTarget(BitmapHeapScanState *node); -extern void BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan); - /* * To avoid leaking too much knowledge about reorderbuffer implementation * details this is implemented in reorderbuffer.c not heapam_visibility.c diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index f344d97f1f..9ec0a1da2b 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -399,7 +399,8 @@ typedef struct TableAmRoutine uint32 flags); void (*scan_rescan_bm) (TableScanDesc scan, TIDBitmap *tbm, - ParallelBitmapHeapState *pstate, dsa_area *dsa); + ParallelBitmapHeapState *pstate, dsa_area *dsa, + int pf_maximum); /* * Release resources and deallocate scan. If TableScanDesc.temp_snap, @@ -801,17 +802,6 @@ typedef struct TableAmRoutine * lossy_pages is incremented if the bitmap is lossy for the selected * block; otherwise, exact_pages is incremented. * - * XXX: Currently this may only be implemented if the AM uses md.c as its - * storage manager, and uses ItemPointer->ip_blkid in a manner that maps - * blockids directly to the underlying storage. nodeBitmapHeapscan.c - * performs prefetching directly using that interface. This probably - * needs to be rectified at a later point. - * - * XXX: Currently this may only be implemented if the AM uses the - * visibilitymap, as nodeBitmapHeapscan.c unconditionally accesses it to - * perform prefetching. This probably needs to be rectified at a later - * point. - * * Optional callback, but either both scan_bitmap_next_block and * scan_bitmap_next_tuple need to exist, or neither. */ @@ -950,7 +940,7 @@ table_beginscan_strat(Relation rel, Snapshot snapshot, */ static inline TableScanDesc table_beginscan_bm(TableScanDesc scan, Relation rel, Snapshot snapshot, - uint32 extra_flags, TIDBitmap *tbm, + uint32 extra_flags, int pf_maximum, TIDBitmap *tbm, ParallelBitmapHeapState *pstate, dsa_area *dsa) { uint32 flags = SO_TYPE_BITMAPSCAN | SO_ALLOW_PAGEMODE | extra_flags; @@ -962,7 +952,7 @@ table_beginscan_bm(TableScanDesc scan, Relation rel, Snapshot snapshot, if (!scan) scan = rel->rd_tableam->scan_begin_bm(rel, snapshot, flags); - scan->rs_rd->rd_tableam->scan_rescan_bm(scan, tbm, pstate, dsa); + scan->rs_rd->rd_tableam->scan_rescan_bm(scan, tbm, pstate, dsa, pf_maximum); return scan; } diff --git a/src/include/executor/nodeBitmapHeapscan.h b/src/include/executor/nodeBitmapHeapscan.h index 7064f54686..028081db32 100644 --- a/src/include/executor/nodeBitmapHeapscan.h +++ b/src/include/executor/nodeBitmapHeapscan.h @@ -29,6 +29,13 @@ extern void ExecBitmapHeapReInitializeDSM(BitmapHeapScanState *node, extern void ExecBitmapHeapInitializeWorker(BitmapHeapScanState *node, ParallelWorkerContext *pwcxt); +typedef struct BitmapHeapIterator +{ + struct TBMIterator *serial; + struct TBMSharedIterator *parallel; + bool exhausted; +} BitmapHeapIterator; + extern TBMIterateResult *bhs_iterate(BitmapHeapIterator *iterator); extern void bhs_begin_iterate(BitmapHeapIterator *iterator, TIDBitmap *tbm, dsa_area *dsa, dsa_pointer dsp); diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 714eb3e534..713de7d50f 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1787,30 +1787,17 @@ typedef struct ParallelBitmapHeapState ConditionVariable cv; } ParallelBitmapHeapState; -typedef struct BitmapHeapIterator -{ - struct TBMIterator *serial; - struct TBMSharedIterator *parallel; - bool exhausted; -} BitmapHeapIterator; - /* ---------------- * BitmapHeapScanState information * * bitmapqualorig execution state for bitmapqualorig expressions * tbm bitmap obtained from child index scan(s) - * pvmbuffer buffer for visibility-map lookups of prefetched pages * exact_pages total number of exact pages retrieved * lossy_pages total number of lossy pages retrieved - * pf_iterator for prefetching ahead of current page - * prefetch_pages # pages prefetch iterator is ahead of current - * prefetch_target current target prefetch distance - * prefetch_maximum maximum value for prefetch_target * initialized is node is ready to iterate * pstate shared state for parallel bitmap scan * recheck do current page's tuples need recheck * blockno used to validate pf and current block in sync - * pfblockno used to validate pf stays ahead of current block * ---------------- */ typedef struct BitmapHeapScanState @@ -1818,18 +1805,12 @@ typedef struct BitmapHeapScanState ScanState ss; /* its first field is NodeTag */ ExprState *bitmapqualorig; TIDBitmap *tbm; - Buffer pvmbuffer; long exact_pages; long lossy_pages; - int prefetch_pages; - int prefetch_target; - int prefetch_maximum; bool initialized; - BitmapHeapIterator pf_iterator; ParallelBitmapHeapState *pstate; bool recheck; BlockNumber blockno; - BlockNumber pfblockno; } BitmapHeapScanState; /* ---------------- -- 2.40.1 --e3xl7h75mzefinno Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0017-Move-BitmapHeapScan-initialization-to-helper.patch" ^ permalink raw reply [nested|flat] 10+ messages in thread
* [PATCH v19 19/21] Push BitmapHeapScan prefetch code into heap AM @ 2024-04-06 20:21 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 10+ messages in thread From: Melanie Plageman @ 2024-04-06 20:21 UTC (permalink / raw) An earlier commit eliminated a table AM layering violation for the current block in a BitmapHeapScan but left the violation in BitmapHeapScan prefetching code. To resolve this, move and manage all state used for prefetching entirely into the heap AM. Author: Melanie Plageman Discussion: https://postgr.es/m/CAAKRu_ZwCwWFeL_H3ia26bP2e7HiKLWt0ZmGXPVwPO6uXq0vaA%40mail.gmail.com --- src/backend/access/heap/heapam.c | 38 ++++++- src/backend/access/heap/heapam_handler.c | 125 +++++++++++++++------- src/backend/executor/nodeBitmapHeapscan.c | 97 +++-------------- src/include/access/heapam.h | 29 ++++- src/include/access/tableam.h | 18 +--- src/include/nodes/execnodes.h | 12 --- 6 files changed, 167 insertions(+), 152 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index a45fdb50285..71f93bfbb1d 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -1065,6 +1065,14 @@ heap_beginscan_bm(Relation relation, Snapshot snapshot, uint32 flags) scan->vmbuffer = InvalidBuffer; scan->empty_tuples_pending = 0; + /* + * The rest of the BitmapHeapScanDesc members related to prefetching will + * be initialized in heap_rescan_bm(). + */ + scan->pvmbuffer = InvalidBuffer; + scan->prefetch_iterator.serial = NULL; + scan->prefetch_iterator.parallel = NULL; + return (TableScanDesc) scan; } @@ -1108,7 +1116,8 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, void heap_rescan_bm(TableScanDesc sscan, TIDBitmap *tbm, - ParallelBitmapHeapState *pstate, dsa_area *dsa) + ParallelBitmapHeapState *pstate, dsa_area *dsa, + int prefetch_maximum) { BitmapHeapScanDesc scan = (BitmapHeapScanDesc) sscan; @@ -1122,6 +1131,17 @@ heap_rescan_bm(TableScanDesc sscan, TIDBitmap *tbm, scan->empty_tuples_pending = 0; unified_tbm_end_iterate(&scan->iterator); + if (BufferIsValid(scan->pvmbuffer)) + ReleaseBuffer(scan->pvmbuffer); + + unified_tbm_end_iterate(&scan->prefetch_iterator); + + scan->prefetch_maximum = prefetch_maximum; + scan->pstate = pstate; + scan->prefetch_target = -1; + scan->prefetch_pages = 0; + scan->pfblockno = InvalidBlockNumber; + /* * reinitialize heap scan descriptor */ @@ -1131,6 +1151,17 @@ heap_rescan_bm(TableScanDesc sscan, TIDBitmap *tbm, pstate ? pstate->tbmiterator : InvalidDsaPointer); + +#ifdef USE_PREFETCH + if (prefetch_maximum > 0) + { + unified_tbm_begin_iterate(&scan->prefetch_iterator, tbm, dsa, + pstate ? + pstate->prefetch_iterator : + InvalidDsaPointer); + } +#endif /* USE_PREFETCH */ + } @@ -1183,6 +1214,11 @@ heap_endscan_bm(TableScanDesc sscan) unified_tbm_end_iterate(&scan->iterator); + if (BufferIsValid(scan->pvmbuffer)) + ReleaseBuffer(scan->pvmbuffer); + + unified_tbm_end_iterate(&scan->prefetch_iterator); + /* * decrement relation reference count and free scan descriptor storage */ diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 85564fef2e0..77888cdc90d 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -60,6 +60,9 @@ static bool SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, OffsetNumber tupoffset); static BlockNumber heapam_scan_get_blocks_done(HeapScanDesc hscan); +static inline void BitmapAdjustPrefetchIterator(BitmapHeapScanDesc scan); +static inline void BitmapAdjustPrefetchTarget(BitmapHeapScanDesc scan); +static inline void BitmapPrefetch(BitmapHeapScanDesc scan); static const TableAmRoutine heapam_methods; @@ -2187,26 +2190,26 @@ heapam_estimate_rel_size(Relation rel, int32 *attr_widths, * iterator in prefetch_pages. For each block the main iterator returns, we * decrement prefetch_pages. */ -void -BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) +static inline void +BitmapAdjustPrefetchIterator(BitmapHeapScanDesc scan) { #ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - UnifiedTBMIterator *prefetch_iterator = &node->prefetch_iterator; + ParallelBitmapHeapState *pstate = scan->pstate; + UnifiedTBMIterator *prefetch_iterator = &scan->prefetch_iterator; TBMIterateResult *tbmpre; if (pstate == NULL) { - if (node->prefetch_pages > 0) + if (scan->prefetch_pages > 0) { /* The main iterator has closed the distance by one page */ - node->prefetch_pages--; + scan->prefetch_pages--; } else if (!prefetch_iterator->exhausted) { /* Do not let the prefetch iterator get behind the main one */ tbmpre = unified_tbm_iterate(prefetch_iterator); - node->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; + scan->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; } return; } @@ -2220,7 +2223,7 @@ BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) * Note that moving the call site of BitmapAdjustPrefetchIterator() * exacerbates the effects of this bug. */ - if (node->prefetch_maximum > 0) + if (scan->prefetch_maximum > 0) { SpinLockAcquire(&pstate->mutex); if (pstate->prefetch_pages > 0) @@ -2244,7 +2247,7 @@ BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) if (!prefetch_iterator->exhausted) { tbmpre = unified_tbm_iterate(prefetch_iterator); - node->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; + scan->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; } } } @@ -2259,33 +2262,33 @@ BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) * page/tuple, then to one after the second tuple is fetched, then * it doubles as later pages are fetched. */ -void -BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) +static inline void +BitmapAdjustPrefetchTarget(BitmapHeapScanDesc scan) { #ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; + ParallelBitmapHeapState *pstate = scan->pstate; if (pstate == NULL) { - if (node->prefetch_target >= node->prefetch_maximum) + if (scan->prefetch_target >= scan->prefetch_maximum) /* don't increase any further */ ; - else if (node->prefetch_target >= node->prefetch_maximum / 2) - node->prefetch_target = node->prefetch_maximum; - else if (node->prefetch_target > 0) - node->prefetch_target *= 2; + else if (scan->prefetch_target >= scan->prefetch_maximum / 2) + scan->prefetch_target = scan->prefetch_maximum; + else if (scan->prefetch_target > 0) + scan->prefetch_target *= 2; else - node->prefetch_target++; + scan->prefetch_target++; return; } /* Do an unlocked check first to save spinlock acquisitions. */ - if (pstate->prefetch_target < node->prefetch_maximum) + if (pstate->prefetch_target < scan->prefetch_maximum) { SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target >= node->prefetch_maximum) + if (pstate->prefetch_target >= scan->prefetch_maximum) /* don't increase any further */ ; - else if (pstate->prefetch_target >= node->prefetch_maximum / 2) - pstate->prefetch_target = node->prefetch_maximum; + else if (pstate->prefetch_target >= scan->prefetch_maximum / 2) + pstate->prefetch_target = scan->prefetch_maximum; else if (pstate->prefetch_target > 0) pstate->prefetch_target *= 2; else @@ -2298,18 +2301,19 @@ BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) /* * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target */ -void -BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) +static inline void +BitmapPrefetch(BitmapHeapScanDesc scan) { #ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; - UnifiedTBMIterator *prefetch_iterator = &node->prefetch_iterator; + ParallelBitmapHeapState *pstate = scan->pstate; + Relation rel = scan->heap_common.rs_base.rs_rd; + UnifiedTBMIterator *prefetch_iterator = &scan->prefetch_iterator; if (pstate == NULL) { if (!prefetch_iterator->exhausted) { - while (node->prefetch_pages < node->prefetch_target) + while (scan->prefetch_pages < scan->prefetch_target) { TBMIterateResult *tbmpre = unified_tbm_iterate(prefetch_iterator); bool skip_fetch; @@ -2320,8 +2324,8 @@ BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) unified_tbm_end_iterate(prefetch_iterator); break; } - node->prefetch_pages++; - node->pfblockno = tbmpre->blockno; + scan->prefetch_pages++; + scan->pfblockno = tbmpre->blockno; /* * If we expect not to have to actually read this heap page, @@ -2329,14 +2333,14 @@ BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) * logic normally. (Would it be better not to increment * prefetch_pages?) */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLES) && + skip_fetch = (!(scan->heap_common.rs_base.rs_flags & SO_NEED_TUPLES) && !tbmpre->recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, + VM_ALL_VISIBLE(rel, tbmpre->blockno, - &node->pvmbuffer)); + &scan->pvmbuffer)); if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre->blockno); + PrefetchBuffer(rel, MAIN_FORKNUM, tbmpre->blockno); } } @@ -2376,17 +2380,17 @@ BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) break; } - node->pfblockno = tbmpre->blockno; + scan->pfblockno = tbmpre->blockno; /* As above, skip prefetch if we expect not to need page */ - skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLES) && + skip_fetch = (!(scan->heap_common.rs_base.rs_flags & SO_NEED_TUPLES) && !tbmpre->recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, + VM_ALL_VISIBLE(rel, tbmpre->blockno, - &node->pvmbuffer)); + &scan->pvmbuffer)); if (!skip_fetch) - PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre->blockno); + PrefetchBuffer(rel, MAIN_FORKNUM, tbmpre->blockno); } } } @@ -2419,6 +2423,8 @@ heapam_scan_bitmap_next_block(TableScanDesc sscan, *blockno = InvalidBlockNumber; *recheck = true; + BitmapAdjustPrefetchIterator(scan); + do { CHECK_FOR_INTERRUPTS(); @@ -2559,6 +2565,19 @@ heapam_scan_bitmap_next_block(TableScanDesc sscan, else (*exact_pages)++; + + /* + * If serial, we can error out if the the prefetch block doesn't stay + * ahead of the current block. + */ + if (scan->pstate == NULL && + !scan->prefetch_iterator.exhausted && + scan->pfblockno < block) + elog(ERROR, "prefetch and main iterators are out of sync"); + + /* Adjust the prefetch target */ + BitmapAdjustPrefetchTarget(scan); + /* * Return true to indicate that a valid block was found and the bitmap is * not exhausted. If there are no visible tuples on this page, @@ -2595,6 +2614,36 @@ heapam_scan_bitmap_next_tuple(TableScanDesc sscan, if (hscan->rs_cindex < 0 || hscan->rs_cindex >= hscan->rs_ntuples) return false; +#ifdef USE_PREFETCH + + /* + * Try to prefetch at least a few pages even before we get to the second + * page if we don't stop reading after the first tuple. + */ + if (!scan->pstate) + { + if (scan->prefetch_target < scan->prefetch_maximum) + scan->prefetch_target++; + } + else if (scan->pstate->prefetch_target < scan->prefetch_maximum) + { + /* take spinlock while updating shared state */ + SpinLockAcquire(&scan->pstate->mutex); + if (scan->pstate->prefetch_target < scan->prefetch_maximum) + scan->pstate->prefetch_target++; + SpinLockRelease(&scan->pstate->mutex); + } + + /* + * We issue prefetch requests *after* fetching the current page to try to + * avoid having prefetching interfere with the main I/O. Also, this should + * happen only when we have determined there is still something to do on + * the current page, else we may uselessly prefetch the same page we are + * just about to request for real. + */ + BitmapPrefetch(scan); +#endif /* USE_PREFETCH */ + targoffset = hscan->rs_vistuples[hscan->rs_cindex]; page = BufferGetPage(hscan->rs_cbuf); lp = PageGetItemId(page, targoffset); diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index fda5645e4ae..11945ab9c42 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -69,7 +69,6 @@ BitmapHeapNext(BitmapHeapScanState *node) TIDBitmap *tbm; TupleTableSlot *slot; ParallelBitmapHeapState *pstate = node->pstate; - dsa_area *dsa = node->ss.ps.state->es_query_dsa; /* * extract necessary information from index scan node @@ -93,7 +92,9 @@ BitmapHeapNext(BitmapHeapScanState *node) */ if (!node->initialized) { + Relation rel = node->ss.ss_currentRelation; bool need_tuples = false; + int prefetch_maximum = 0; /* * The leader will immediately come out of the function, but others @@ -103,6 +104,14 @@ BitmapHeapNext(BitmapHeapScanState *node) bool init_shared_state = node->pstate ? BitmapShouldInitializeSharedState(node->pstate) : false; + /* + * Maximum number of prefetches for the tablespace if configured, + * otherwise the current value of the effective_io_concurrency GUC. + */ +#ifdef USE_PREFETCH + prefetch_maximum = get_tablespace_io_concurrency(rel->rd_rel->reltablespace); +#endif + /* * Only serial bitmap table scans and the parallel leader in a * parallel bitmap table scan should build the bitmap. @@ -124,7 +133,7 @@ BitmapHeapNext(BitmapHeapScanState *node) */ pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (prefetch_maximum > 0) { pstate->prefetch_iterator = tbm_prepare_shared_iterate(tbm); @@ -154,22 +163,13 @@ BitmapHeapNext(BitmapHeapScanState *node) node->ss.ss_currentRelation, node->ss.ps.state->es_snapshot, need_tuples, + prefetch_maximum, node->tbm, node->pstate, node->ss.ps.state->es_query_dsa); node->ss.ss_currentScanDesc = scan; -#ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) - { - unified_tbm_begin_iterate(&node->prefetch_iterator, tbm, dsa, - pstate ? - pstate->prefetch_iterator : - InvalidDsaPointer); - } -#endif /* USE_PREFETCH */ - node->initialized = true; goto new_page; @@ -185,37 +185,6 @@ BitmapHeapNext(BitmapHeapScanState *node) CHECK_FOR_INTERRUPTS(); -#ifdef USE_PREFETCH - - /* - * Try to prefetch at least a few pages even before we get to the - * second page if we don't stop reading after the first tuple. - */ - if (!pstate) - { - if (node->prefetch_target < node->prefetch_maximum) - node->prefetch_target++; - } - else if (pstate->prefetch_target < node->prefetch_maximum) - { - /* take spinlock while updating shared state */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target < node->prefetch_maximum) - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ - - /* - * We issue prefetch requests *after* fetching the current page to - * try to avoid having prefetching interfere with the main I/O. - * Also, this should happen only when we have determined there is - * still something to do on the current page, else we may - * uselessly prefetch the same page we are just about to request - * for real. - */ - BitmapPrefetch(node, scan); - /* * If we are using lossy info, we have to recheck the qual * conditions at every tuple. @@ -238,23 +207,9 @@ BitmapHeapNext(BitmapHeapScanState *node) new_page: - BitmapAdjustPrefetchIterator(node); - if (!table_scan_bitmap_next_block(scan, &node->blockno, &node->recheck, &node->lossy_pages, &node->exact_pages)) break; - - /* - * If serial, we can error out if the the prefetch block doesn't stay - * ahead of the current block. - */ - if (node->pstate == NULL && - !node->prefetch_iterator.exhausted && - node->pfblockno < node->blockno) - elog(ERROR, "prefetch and main iterators are out of sync"); - - /* Adjust the prefetch target */ - BitmapAdjustPrefetchTarget(node); } /* @@ -319,22 +274,13 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) { PlanState *outerPlan = outerPlanState(node); - /* release bitmaps and buffers if any */ - if (node->prefetch_iterator.exhausted) - unified_tbm_end_iterate(&node->prefetch_iterator); + /* release bitmaps if any */ if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); node->tbm = NULL; node->initialized = false; - node->pvmbuffer = InvalidBuffer; node->recheck = true; node->blockno = InvalidBlockNumber; - node->pfblockno = InvalidBlockNumber; - /* Only used for serial BHS */ - node->prefetch_pages = 0; - node->prefetch_target = -1; ExecScanReScan(&node->ss); @@ -373,14 +319,10 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) table_endscan_bm(scanDesc); /* - * release bitmaps and buffers if any + * release bitmaps if any */ - if (!node->prefetch_iterator.exhausted) - unified_tbm_end_iterate(&node->prefetch_iterator); if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); } /* ---------------------------------------------------------------- @@ -413,16 +355,12 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ps.ExecProcNode = ExecBitmapHeapScan; scanstate->tbm = NULL; - scanstate->pvmbuffer = InvalidBuffer; scanstate->exact_pages = 0; scanstate->lossy_pages = 0; - scanstate->prefetch_pages = 0; - scanstate->prefetch_target = -1; scanstate->initialized = false; scanstate->pstate = NULL; scanstate->recheck = true; scanstate->blockno = InvalidBlockNumber; - scanstate->pfblockno = InvalidBlockNumber; /* * Miscellaneous initialization @@ -462,13 +400,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->bitmapqualorig = ExecInitQual(node->bitmapqualorig, (PlanState *) scanstate); - /* - * Maximum number of prefetches for the tablespace if configured, - * otherwise the current value of the effective_io_concurrency GUC. - */ - scanstate->prefetch_maximum = - get_tablespace_io_concurrency(currentRelation->rd_rel->reltablespace); - scanstate->ss.ss_currentRelation = currentRelation; /* diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index f0cd0041460..b17f0e8b4e6 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -92,6 +92,10 @@ typedef struct BitmapHeapScanDescData * Members common to Parallel and Serial BitmapHeapScan */ UnifiedTBMIterator iterator; + UnifiedTBMIterator prefetch_iterator; + + /* maximum value for prefetch_target */ + int prefetch_maximum; /* * These fields are only used for bitmap scans for the "skip fetch" @@ -101,7 +105,26 @@ typedef struct BitmapHeapScanDescData * to return. They are common to parallel and serial BitmapHeapScans */ Buffer vmbuffer; + /* buffer for visibility-map lookups of prefetched pages */ + Buffer pvmbuffer; int empty_tuples_pending; + + /* + * Parallel-only members + */ + + struct ParallelBitmapHeapState *pstate; + + /* + * Serial-only members + */ + + /* Current target for prefetch distance */ + int prefetch_target; + /* # pages prefetch iterator is ahead of current */ + int prefetch_pages; + /* used to validate prefetch block stays ahead of current block */ + BlockNumber pfblockno; } BitmapHeapScanDescData; typedef struct BitmapHeapScanDescData *BitmapHeapScanDesc; @@ -305,7 +328,8 @@ extern bool heap_getnextslot_tidrange(TableScanDesc sscan, extern TableScanDesc heap_beginscan_bm(Relation relation, Snapshot snapshot, uint32 flags); extern void heap_endscan_bm(TableScanDesc scan); extern void heap_rescan_bm(TableScanDesc sscan, TIDBitmap *tbm, - ParallelBitmapHeapState *pstate, dsa_area *dsa); + ParallelBitmapHeapState *pstate, dsa_area *dsa, + int prefetch_maximum); extern bool heap_fetch(Relation relation, Snapshot snapshot, HeapTuple tuple, Buffer *userbuf, bool keep_buf); extern bool heap_hot_search_buffer(ItemPointer tid, Relation relation, @@ -422,9 +446,6 @@ extern bool heapam_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, double *liverows, double *deadrows, TupleTableSlot *slot); -extern void BitmapAdjustPrefetchIterator(BitmapHeapScanState *node); -extern void BitmapAdjustPrefetchTarget(BitmapHeapScanState *node); -extern void BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan); /* * To avoid leaking too much knowledge about reorderbuffer implementation diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index aebcecad30d..06530536897 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -368,7 +368,8 @@ typedef struct TableAmRoutine uint32 flags); void (*scan_rescan_bm) (TableScanDesc scan, TIDBitmap *tbm, - ParallelBitmapHeapState *pstate, dsa_area *dsa); + ParallelBitmapHeapState *pstate, dsa_area *dsa, + int prefetch_maximum); /* * Release resources and deallocate scan. @@ -805,17 +806,6 @@ typedef struct TableAmRoutine * lossy_pages is incremented if the bitmap is lossy for the selected * block; otherwise, exact_pages is incremented. * - * XXX: Currently this may only be implemented if the AM uses md.c as its - * storage manager, and uses ItemPointer->ip_blkid in a manner that maps - * blockids directly to the underlying storage. nodeBitmapHeapscan.c - * performs prefetching directly using that interface. This probably - * needs to be rectified at a later point. - * - * XXX: Currently this may only be implemented if the AM uses the - * visibilitymap, as nodeBitmapHeapscan.c unconditionally accesses it to - * perform prefetching. This probably needs to be rectified at a later - * point. - * * Optional callback, but either both scan_bitmap_next_block and * scan_bitmap_next_tuple need to exist, or neither. */ @@ -954,7 +944,7 @@ table_beginscan_strat(Relation rel, Snapshot snapshot, */ static inline TableScanDesc table_beginscan_bm(TableScanDesc scan, Relation rel, Snapshot snapshot, - bool need_tuple, TIDBitmap *tbm, + bool need_tuple, int prefetch_maximum, TIDBitmap *tbm, ParallelBitmapHeapState *pstate, dsa_area *dsa) { uint32 flags = SO_TYPE_BITMAPSCAN | SO_ALLOW_PAGEMODE; @@ -969,7 +959,7 @@ table_beginscan_bm(TableScanDesc scan, Relation rel, Snapshot snapshot, if (!scan) scan = rel->rd_tableam->scan_begin_bm(rel, snapshot, flags); - scan->rs_rd->rd_tableam->scan_rescan_bm(scan, tbm, pstate, dsa); + scan->rs_rd->rd_tableam->scan_rescan_bm(scan, tbm, pstate, dsa, prefetch_maximum); return scan; } diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 6693cf66c77..713de7d50f6 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1792,18 +1792,12 @@ typedef struct ParallelBitmapHeapState * * bitmapqualorig execution state for bitmapqualorig expressions * tbm bitmap obtained from child index scan(s) - * pvmbuffer buffer for visibility-map lookups of prefetched pages * exact_pages total number of exact pages retrieved * lossy_pages total number of lossy pages retrieved - * prefetch_iterator for prefetching ahead of current page - * prefetch_pages # pages prefetch iterator is ahead of current - * prefetch_target current target prefetch distance - * prefetch_maximum maximum value for prefetch_target * initialized is node is ready to iterate * pstate shared state for parallel bitmap scan * recheck do current page's tuples need recheck * blockno used to validate pf and current block in sync - * pfblockno used to validate pf stays ahead of current block * ---------------- */ typedef struct BitmapHeapScanState @@ -1811,18 +1805,12 @@ typedef struct BitmapHeapScanState ScanState ss; /* its first field is NodeTag */ ExprState *bitmapqualorig; TIDBitmap *tbm; - Buffer pvmbuffer; long exact_pages; long lossy_pages; - int prefetch_pages; - int prefetch_target; - int prefetch_maximum; bool initialized; - UnifiedTBMIterator prefetch_iterator; ParallelBitmapHeapState *pstate; bool recheck; BlockNumber blockno; - BlockNumber pfblockno; } BitmapHeapScanState; /* ---------------- -- 2.40.1 --clv5reqgj4vdihqz Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v19-0020-Move-BitmapHeapScan-initialization-to-helper.patch" ^ permalink raw reply [nested|flat] 10+ messages in thread
end of thread, other threads:[~2024-04-06 20:21 UTC | newest] Thread overview: 10+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-11-11 22:40 [PATCH 1/1] resownerbench Heikki Linnakangas <[email protected]> 2024-03-22 13:42 [PATCH v8 13/17] Push BitmapHeapScan prefetch code into heapam.c Melanie Plageman <[email protected]> 2024-03-22 13:42 [PATCH v11 14/17] Push BitmapHeapScan prefetch code into heapam.c Melanie Plageman <[email protected]> 2024-03-22 13:42 [PATCH v13 14/16] Push BitmapHeapScan prefetch code into heapam.c Melanie Plageman <[email protected]> 2024-03-22 13:42 [PATCH v15 11/13] Push BitmapHeapScan prefetch code into heapam.c Melanie Plageman <[email protected]> 2024-03-22 13:42 [PATCH v10 14/17] Push BitmapHeapScan prefetch code into heapam.c Melanie Plageman <[email protected]> 2024-03-22 13:42 [PATCH v12 14/17] Push BitmapHeapScan prefetch code into heapam.c Melanie Plageman <[email protected]> 2024-03-22 13:42 [PATCH v9 13/17] Push BitmapHeapScan prefetch code into heapam.c Melanie Plageman <[email protected]> 2024-04-05 22:48 [PATCH v16 16/18] Push BitmapHeapScan prefetch code into heap AM Melanie Plageman <[email protected]> 2024-04-06 20:21 [PATCH v19 19/21] Push BitmapHeapScan prefetch code into heap AM Melanie Plageman <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox