public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v5a 6/7] Vacuum first pass uses Streaming Read interface 7+ messages / 2 participants [nested] [flat]
* [PATCH v5a 6/7] Vacuum first pass uses Streaming Read interface @ 2023-12-31 16:29 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Melanie Plageman @ 2023-12-31 16:29 UTC (permalink / raw) Now vacuum's first pass, which HOT prunes and records the TIDs of non-removable dead tuples, uses the streaming read API by implementing a streaming read callback which invokes heap_vac_scan_get_next_block(). --- src/backend/access/heap/vacuumlazy.c | 79 +++++++++++++++++++++------- 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 65d257aab83..fbbc87938e4 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -54,6 +54,7 @@ #include "storage/bufmgr.h" #include "storage/freespace.h" #include "storage/lmgr.h" +#include "storage/streaming_read.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" @@ -168,7 +169,12 @@ typedef struct LVRelState char *relnamespace; char *relname; char *indname; /* Current index name */ - BlockNumber blkno; /* used only for heap operations */ + + /* + * The current block being processed by vacuum. Used only for heap + * operations. Primarily for error reporting and logging. + */ + BlockNumber blkno; OffsetNumber offnum; /* used only for heap operations */ VacErrPhase phase; bool verbose; /* VACUUM VERBOSE? */ @@ -189,6 +195,12 @@ typedef struct LVRelState BlockNumber missed_dead_pages; /* # pages with missed dead tuples */ BlockNumber nonempty_pages; /* actually, last nonempty page + 1 */ + /* + * The most recent block submitted in the streaming read callback by the + * first vacuum pass. + */ + BlockNumber blkno_prefetch; + /* Statistics output by us, for table */ double new_rel_tuples; /* new estimated total # of tuples */ double new_live_tuples; /* new estimated total # of live tuples */ @@ -232,7 +244,7 @@ typedef struct LVSavedErrInfo /* non-export function prototypes */ static void lazy_scan_heap(LVRelState *vacrel); -static bool heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, +static void heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, BlockNumber *blkno, bool *all_visible_according_to_vm); static bool lazy_scan_new_or_empty(LVRelState *vacrel, Buffer buf, @@ -416,6 +428,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->nonempty_pages = 0; /* dead_items_alloc allocates vacrel->dead_items later on */ + /* relies on InvalidBlockNumber overflowing to 0 */ + vacrel->blkno_prefetch = InvalidBlockNumber; + /* Allocate/initialize output statistics state */ vacrel->new_rel_tuples = 0; vacrel->new_live_tuples = 0; @@ -776,6 +791,22 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } +static BlockNumber +vacuum_scan_pgsr_next(PgStreamingRead *pgsr, + void *pgsr_private, void *per_buffer_data) +{ + LVRelState *vacrel = pgsr_private; + bool *all_visible_according_to_vm = per_buffer_data; + + vacrel->blkno_prefetch++; + + heap_vac_scan_get_next_block(vacrel, + vacrel->blkno_prefetch, &vacrel->blkno_prefetch, + all_visible_according_to_vm); + + return vacrel->blkno_prefetch; +} + /* * lazy_scan_heap() -- workhorse function for VACUUM * @@ -815,12 +846,11 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, static void lazy_scan_heap(LVRelState *vacrel) { + Buffer buf; BlockNumber rel_pages = vacrel->rel_pages, next_fsm_block_to_vacuum = 0; - bool all_visible_according_to_vm; + bool *all_visible_according_to_vm; - /* relies on InvalidBlockNumber overflowing to 0 */ - BlockNumber blkno = InvalidBlockNumber; VacDeadItems *dead_items = vacrel->dead_items; const int initprog_index[] = { PROGRESS_VACUUM_PHASE, @@ -828,6 +858,11 @@ lazy_scan_heap(LVRelState *vacrel) PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + PgStreamingRead *pgsr; + + pgsr = pg_streaming_read_buffer_alloc(PGSR_FLAG_MAINTENANCE, vacrel, + sizeof(bool), vacrel->bstrategy, BMR_REL(vacrel->rel), + MAIN_FORKNUM, vacuum_scan_pgsr_next); /* Report that we're scanning the heap, advertising total # of blocks */ initprog_val[0] = PROGRESS_VACUUM_PHASE_SCAN_HEAP; @@ -838,13 +873,19 @@ lazy_scan_heap(LVRelState *vacrel) vacrel->skip.next_unskippable_block = InvalidBlockNumber; vacrel->skip.vmbuffer = InvalidBuffer; - while (heap_vac_scan_get_next_block(vacrel, blkno + 1, - &blkno, &all_visible_according_to_vm)) + while (BufferIsValid(buf = + pg_streaming_read_buffer_get_next(pgsr, (void **) &all_visible_according_to_vm))) { - Buffer buf; Page page; bool has_lpdead_items; bool got_cleanup_lock = false; + BlockNumber blkno; + + vacrel->blkno = blkno = BufferGetBlockNumber(buf); + + CheckBufferIsPinnedOnce(buf); + + page = BufferGetPage(buf); vacrel->scanned_pages++; @@ -912,9 +953,6 @@ lazy_scan_heap(LVRelState *vacrel) */ visibilitymap_pin(vacrel->rel, blkno, &vacrel->skip.vmbuffer); - buf = ReadBufferExtended(vacrel->rel, MAIN_FORKNUM, blkno, RBM_NORMAL, - vacrel->bstrategy); - page = BufferGetPage(buf); /* * We need a buffer cleanup lock to prune HOT chains and defragment @@ -970,7 +1008,7 @@ lazy_scan_heap(LVRelState *vacrel) */ if (got_cleanup_lock) lazy_scan_prune(vacrel, buf, blkno, page, - all_visible_according_to_vm, + *all_visible_according_to_vm, &has_lpdead_items); /* @@ -1027,7 +1065,7 @@ lazy_scan_heap(LVRelState *vacrel) } /* report that everything is now scanned */ - pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, vacrel->rel_pages); /* now we can compute the new value for pg_class.reltuples */ vacrel->new_live_tuples = vac_estimate_reltuples(vacrel->rel, rel_pages, @@ -1042,6 +1080,8 @@ lazy_scan_heap(LVRelState *vacrel) Max(vacrel->new_live_tuples, 0) + vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; + pg_streaming_read_free(pgsr); + /* * Do index vacuuming (call each index's ambulkdelete routine), then do * related heap vacuuming @@ -1053,11 +1093,11 @@ lazy_scan_heap(LVRelState *vacrel) * Vacuum the remainder of the Free Space Map. We must do this whether or * not there were indexes, and whether or not we bypassed index vacuuming. */ - if (blkno > next_fsm_block_to_vacuum) - FreeSpaceMapVacuumRange(vacrel->rel, next_fsm_block_to_vacuum, blkno); + if (vacrel->rel_pages > next_fsm_block_to_vacuum) + FreeSpaceMapVacuumRange(vacrel->rel, next_fsm_block_to_vacuum, vacrel->rel_pages); /* report all blocks vacuumed */ - pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, vacrel->rel_pages); /* Do final index cleanup (call each index's amvacuumcleanup routine) */ if (vacrel->nindexes > 0 && vacrel->do_index_cleanup) @@ -1090,7 +1130,7 @@ lazy_scan_heap(LVRelState *vacrel) * * The block number and visibility status of the next block to process are set * in blkno and all_visible_according_to_vm. heap_vac_scan_get_next_block() - * returns false if there are no further blocks to process. + * sets blkno to InvalidBlockNumber if there are no further blocks to process. * * vacrel is an in/out parameter here; vacuum options and information about the * relation are read and vacrel->skippedallvis is set to ensure we don't @@ -1110,7 +1150,7 @@ lazy_scan_heap(LVRelState *vacrel) * older XIDs/MXIDs. The vacrel->skippedallvis flag will be set here when the * choice to skip such a range is actually made, making everything safe.) */ -static bool +static void heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, BlockNumber *blkno, bool *all_visible_according_to_vm) { @@ -1119,7 +1159,7 @@ heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, if (next_block >= vacrel->rel_pages) { *blkno = InvalidBlockNumber; - return false; + return; } if (vacrel->skip.next_unskippable_block == InvalidBlockNumber || @@ -1214,7 +1254,6 @@ heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, *all_visible_according_to_vm = true; *blkno = next_block; - return true; } /* -- 2.40.1 --lb5e4aqx4b6ik5lq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5a-0007-Vacuum-second-pass-uses-Streaming-Read-interface.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH v5a 6/7] Vacuum first pass uses Streaming Read interface @ 2023-12-31 16:29 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Melanie Plageman @ 2023-12-31 16:29 UTC (permalink / raw) Now vacuum's first pass, which HOT prunes and records the TIDs of non-removable dead tuples, uses the streaming read API by implementing a streaming read callback which invokes heap_vac_scan_get_next_block(). --- src/backend/access/heap/vacuumlazy.c | 79 +++++++++++++++++++++------- 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 65d257aab83..fbbc87938e4 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -54,6 +54,7 @@ #include "storage/bufmgr.h" #include "storage/freespace.h" #include "storage/lmgr.h" +#include "storage/streaming_read.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" @@ -168,7 +169,12 @@ typedef struct LVRelState char *relnamespace; char *relname; char *indname; /* Current index name */ - BlockNumber blkno; /* used only for heap operations */ + + /* + * The current block being processed by vacuum. Used only for heap + * operations. Primarily for error reporting and logging. + */ + BlockNumber blkno; OffsetNumber offnum; /* used only for heap operations */ VacErrPhase phase; bool verbose; /* VACUUM VERBOSE? */ @@ -189,6 +195,12 @@ typedef struct LVRelState BlockNumber missed_dead_pages; /* # pages with missed dead tuples */ BlockNumber nonempty_pages; /* actually, last nonempty page + 1 */ + /* + * The most recent block submitted in the streaming read callback by the + * first vacuum pass. + */ + BlockNumber blkno_prefetch; + /* Statistics output by us, for table */ double new_rel_tuples; /* new estimated total # of tuples */ double new_live_tuples; /* new estimated total # of live tuples */ @@ -232,7 +244,7 @@ typedef struct LVSavedErrInfo /* non-export function prototypes */ static void lazy_scan_heap(LVRelState *vacrel); -static bool heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, +static void heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, BlockNumber *blkno, bool *all_visible_according_to_vm); static bool lazy_scan_new_or_empty(LVRelState *vacrel, Buffer buf, @@ -416,6 +428,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->nonempty_pages = 0; /* dead_items_alloc allocates vacrel->dead_items later on */ + /* relies on InvalidBlockNumber overflowing to 0 */ + vacrel->blkno_prefetch = InvalidBlockNumber; + /* Allocate/initialize output statistics state */ vacrel->new_rel_tuples = 0; vacrel->new_live_tuples = 0; @@ -776,6 +791,22 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } +static BlockNumber +vacuum_scan_pgsr_next(PgStreamingRead *pgsr, + void *pgsr_private, void *per_buffer_data) +{ + LVRelState *vacrel = pgsr_private; + bool *all_visible_according_to_vm = per_buffer_data; + + vacrel->blkno_prefetch++; + + heap_vac_scan_get_next_block(vacrel, + vacrel->blkno_prefetch, &vacrel->blkno_prefetch, + all_visible_according_to_vm); + + return vacrel->blkno_prefetch; +} + /* * lazy_scan_heap() -- workhorse function for VACUUM * @@ -815,12 +846,11 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, static void lazy_scan_heap(LVRelState *vacrel) { + Buffer buf; BlockNumber rel_pages = vacrel->rel_pages, next_fsm_block_to_vacuum = 0; - bool all_visible_according_to_vm; + bool *all_visible_according_to_vm; - /* relies on InvalidBlockNumber overflowing to 0 */ - BlockNumber blkno = InvalidBlockNumber; VacDeadItems *dead_items = vacrel->dead_items; const int initprog_index[] = { PROGRESS_VACUUM_PHASE, @@ -828,6 +858,11 @@ lazy_scan_heap(LVRelState *vacrel) PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + PgStreamingRead *pgsr; + + pgsr = pg_streaming_read_buffer_alloc(PGSR_FLAG_MAINTENANCE, vacrel, + sizeof(bool), vacrel->bstrategy, BMR_REL(vacrel->rel), + MAIN_FORKNUM, vacuum_scan_pgsr_next); /* Report that we're scanning the heap, advertising total # of blocks */ initprog_val[0] = PROGRESS_VACUUM_PHASE_SCAN_HEAP; @@ -838,13 +873,19 @@ lazy_scan_heap(LVRelState *vacrel) vacrel->skip.next_unskippable_block = InvalidBlockNumber; vacrel->skip.vmbuffer = InvalidBuffer; - while (heap_vac_scan_get_next_block(vacrel, blkno + 1, - &blkno, &all_visible_according_to_vm)) + while (BufferIsValid(buf = + pg_streaming_read_buffer_get_next(pgsr, (void **) &all_visible_according_to_vm))) { - Buffer buf; Page page; bool has_lpdead_items; bool got_cleanup_lock = false; + BlockNumber blkno; + + vacrel->blkno = blkno = BufferGetBlockNumber(buf); + + CheckBufferIsPinnedOnce(buf); + + page = BufferGetPage(buf); vacrel->scanned_pages++; @@ -912,9 +953,6 @@ lazy_scan_heap(LVRelState *vacrel) */ visibilitymap_pin(vacrel->rel, blkno, &vacrel->skip.vmbuffer); - buf = ReadBufferExtended(vacrel->rel, MAIN_FORKNUM, blkno, RBM_NORMAL, - vacrel->bstrategy); - page = BufferGetPage(buf); /* * We need a buffer cleanup lock to prune HOT chains and defragment @@ -970,7 +1008,7 @@ lazy_scan_heap(LVRelState *vacrel) */ if (got_cleanup_lock) lazy_scan_prune(vacrel, buf, blkno, page, - all_visible_according_to_vm, + *all_visible_according_to_vm, &has_lpdead_items); /* @@ -1027,7 +1065,7 @@ lazy_scan_heap(LVRelState *vacrel) } /* report that everything is now scanned */ - pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, vacrel->rel_pages); /* now we can compute the new value for pg_class.reltuples */ vacrel->new_live_tuples = vac_estimate_reltuples(vacrel->rel, rel_pages, @@ -1042,6 +1080,8 @@ lazy_scan_heap(LVRelState *vacrel) Max(vacrel->new_live_tuples, 0) + vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; + pg_streaming_read_free(pgsr); + /* * Do index vacuuming (call each index's ambulkdelete routine), then do * related heap vacuuming @@ -1053,11 +1093,11 @@ lazy_scan_heap(LVRelState *vacrel) * Vacuum the remainder of the Free Space Map. We must do this whether or * not there were indexes, and whether or not we bypassed index vacuuming. */ - if (blkno > next_fsm_block_to_vacuum) - FreeSpaceMapVacuumRange(vacrel->rel, next_fsm_block_to_vacuum, blkno); + if (vacrel->rel_pages > next_fsm_block_to_vacuum) + FreeSpaceMapVacuumRange(vacrel->rel, next_fsm_block_to_vacuum, vacrel->rel_pages); /* report all blocks vacuumed */ - pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, vacrel->rel_pages); /* Do final index cleanup (call each index's amvacuumcleanup routine) */ if (vacrel->nindexes > 0 && vacrel->do_index_cleanup) @@ -1090,7 +1130,7 @@ lazy_scan_heap(LVRelState *vacrel) * * The block number and visibility status of the next block to process are set * in blkno and all_visible_according_to_vm. heap_vac_scan_get_next_block() - * returns false if there are no further blocks to process. + * sets blkno to InvalidBlockNumber if there are no further blocks to process. * * vacrel is an in/out parameter here; vacuum options and information about the * relation are read and vacrel->skippedallvis is set to ensure we don't @@ -1110,7 +1150,7 @@ lazy_scan_heap(LVRelState *vacrel) * older XIDs/MXIDs. The vacrel->skippedallvis flag will be set here when the * choice to skip such a range is actually made, making everything safe.) */ -static bool +static void heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, BlockNumber *blkno, bool *all_visible_according_to_vm) { @@ -1119,7 +1159,7 @@ heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, if (next_block >= vacrel->rel_pages) { *blkno = InvalidBlockNumber; - return false; + return; } if (vacrel->skip.next_unskippable_block == InvalidBlockNumber || @@ -1214,7 +1254,6 @@ heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, *all_visible_according_to_vm = true; *blkno = next_block; - return true; } /* -- 2.40.1 --lb5e4aqx4b6ik5lq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5a-0007-Vacuum-second-pass-uses-Streaming-Read-interface.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH v7 6/7] Vacuum first pass uses Streaming Read interface @ 2023-12-31 16:29 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Melanie Plageman @ 2023-12-31 16:29 UTC (permalink / raw) Now vacuum's first pass, which HOT prunes and records the TIDs of non-removable dead tuples, uses the streaming read API by implementing a streaming read callback which invokes heap_vac_scan_next_block(). --- src/backend/access/heap/vacuumlazy.c | 131 +++++++++++++++++++-------- 1 file changed, 92 insertions(+), 39 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index d2c8f27fc57..d07a2a58b15 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -54,6 +54,7 @@ #include "storage/bufmgr.h" #include "storage/freespace.h" #include "storage/lmgr.h" +#include "storage/streaming_read.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" @@ -168,7 +169,12 @@ typedef struct LVRelState char *relnamespace; char *relname; char *indname; /* Current index name */ - BlockNumber blkno; /* used only for heap operations */ + + /* + * The current block being processed by vacuum. Used only for heap + * operations. Primarily for error reporting and logging. + */ + BlockNumber blkno; OffsetNumber offnum; /* used only for heap operations */ VacErrPhase phase; bool verbose; /* VACUUM VERBOSE? */ @@ -220,6 +226,12 @@ typedef struct LVRelState BlockNumber next_unskippable_block; /* Next unskippable block's visibility status */ bool next_unskippable_allvis; + + /* + * Buffer containing block of VM with visibility information for + * next_unskippable_block. + */ + Buffer next_unskippable_vmbuffer; } next_block_state; } LVRelState; @@ -233,8 +245,7 @@ typedef struct LVSavedErrInfo /* non-export function prototypes */ static void lazy_scan_heap(LVRelState *vacrel); -static bool heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, - BlockNumber *blkno, +static void heap_vac_scan_next_block(LVRelState *vacrel, bool *all_visible_according_to_vm); static bool lazy_scan_new_or_empty(LVRelState *vacrel, Buffer buf, BlockNumber blkno, Page page, @@ -777,6 +788,47 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } +static BlockNumber +vacuum_scan_pgsr_next(PgStreamingRead *pgsr, + void *pgsr_private, void *per_buffer_data) +{ + LVRelState *vacrel = pgsr_private; + bool *all_visible_according_to_vm = per_buffer_data; + + heap_vac_scan_next_block(vacrel, + all_visible_according_to_vm); + + /* + * If there are no further blocks to vacuum in the relation, release the + * vmbuffer. + */ + if (!BlockNumberIsValid(vacrel->next_block_state.current_block) && + BufferIsValid(vacrel->next_block_state.next_unskippable_vmbuffer)) + { + ReleaseBuffer(vacrel->next_block_state.next_unskippable_vmbuffer); + vacrel->next_block_state.next_unskippable_vmbuffer = InvalidBuffer; + } + + return vacrel->next_block_state.current_block; +} + +static inline PgStreamingRead * +vac_scan_pgsr_alloc(LVRelState *vacrel, PgStreamingReadBufferCB next_block_cb) +{ + PgStreamingRead *result = pg_streaming_read_buffer_alloc(PGSR_FLAG_MAINTENANCE, vacrel, + sizeof(bool), vacrel->bstrategy, BMR_REL(vacrel->rel), + MAIN_FORKNUM, next_block_cb); + + /* + * Initialize for first heap_vac_scan_next_block() call. These rely on + * InvalidBlockNumber + 1 = 0 + */ + vacrel->next_block_state.current_block = InvalidBlockNumber; + vacrel->next_block_state.next_unskippable_block = InvalidBlockNumber; + + return result; +} + /* * lazy_scan_heap() -- workhorse function for VACUUM * @@ -816,10 +868,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, static void lazy_scan_heap(LVRelState *vacrel) { + Buffer buf; BlockNumber rel_pages = vacrel->rel_pages, - blkno, next_fsm_block_to_vacuum = 0; - bool all_visible_according_to_vm; + bool *all_visible_according_to_vm; VacDeadItems *dead_items = vacrel->dead_items; Buffer vmbuffer = InvalidBuffer; @@ -830,23 +882,27 @@ lazy_scan_heap(LVRelState *vacrel) }; int64 initprog_val[3]; + PgStreamingRead *pgsr = vac_scan_pgsr_alloc(vacrel, vacuum_scan_pgsr_next); + /* Report that we're scanning the heap, advertising total # of blocks */ initprog_val[0] = PROGRESS_VACUUM_PHASE_SCAN_HEAP; initprog_val[1] = rel_pages; initprog_val[2] = dead_items->max_items; pgstat_progress_update_multi_param(3, initprog_index, initprog_val); - /* Initialize for first heap_vac_scan_next_block() call */ - vacrel->next_block_state.current_block = InvalidBlockNumber; - vacrel->next_block_state.next_unskippable_block = InvalidBlockNumber; - - while (heap_vac_scan_next_block(vacrel, &vmbuffer, - &blkno, &all_visible_according_to_vm)) + while (BufferIsValid(buf = pg_streaming_read_buffer_get_next(pgsr, + (void **) &all_visible_according_to_vm))) { - Buffer buf; Page page; bool has_lpdead_items; bool got_cleanup_lock = false; + BlockNumber blkno; + + vacrel->blkno = blkno = BufferGetBlockNumber(buf); + + CheckBufferIsPinnedOnce(buf); + + page = BufferGetPage(buf); vacrel->scanned_pages++; @@ -914,9 +970,6 @@ lazy_scan_heap(LVRelState *vacrel) */ visibilitymap_pin(vacrel->rel, blkno, &vmbuffer); - buf = ReadBufferExtended(vacrel->rel, MAIN_FORKNUM, blkno, RBM_NORMAL, - vacrel->bstrategy); - page = BufferGetPage(buf); /* * We need a buffer cleanup lock to prune HOT chains and defragment @@ -973,7 +1026,7 @@ lazy_scan_heap(LVRelState *vacrel) */ if (got_cleanup_lock) lazy_scan_prune(vacrel, buf, blkno, page, - vmbuffer, all_visible_according_to_vm, + vmbuffer, *all_visible_according_to_vm, &has_lpdead_items); /* @@ -1030,7 +1083,7 @@ lazy_scan_heap(LVRelState *vacrel) } /* report that everything is now scanned */ - pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, vacrel->rel_pages); /* now we can compute the new value for pg_class.reltuples */ vacrel->new_live_tuples = vac_estimate_reltuples(vacrel->rel, rel_pages, @@ -1045,6 +1098,8 @@ lazy_scan_heap(LVRelState *vacrel) Max(vacrel->new_live_tuples, 0) + vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; + pg_streaming_read_free(pgsr); + /* * Do index vacuuming (call each index's ambulkdelete routine), then do * related heap vacuuming @@ -1056,11 +1111,11 @@ lazy_scan_heap(LVRelState *vacrel) * Vacuum the remainder of the Free Space Map. We must do this whether or * not there were indexes, and whether or not we bypassed index vacuuming. */ - if (blkno > next_fsm_block_to_vacuum) - FreeSpaceMapVacuumRange(vacrel->rel, next_fsm_block_to_vacuum, blkno); + if (vacrel->rel_pages > next_fsm_block_to_vacuum) + FreeSpaceMapVacuumRange(vacrel->rel, next_fsm_block_to_vacuum, vacrel->rel_pages); /* report all blocks vacuumed */ - pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, vacrel->rel_pages); /* Do final index cleanup (call each index's amvacuumcleanup routine) */ if (vacrel->nindexes > 0 && vacrel->do_index_cleanup) @@ -1072,12 +1127,13 @@ lazy_scan_heap(LVRelState *vacrel) * * lazy_scan_heap() calls here every time it needs to get the next block to * prune and vacuum, using the visibility map, vacuum options, and various - * thresholds to skip blocks which do not need to be processed and set blkno to - * the next block that actually needs to be processed. + * thresholds to skip blocks which do not need to be processed and set + * current_block to the next block that actually needs to be processed. * - * The block number and visibility status of the next block to process are set - * in blkno and all_visible_according_to_vm. heap_vac_scan_next_block() - * returns false if there are no further blocks to process. + * The number and visibility status of the next block to process are set in + * vacrel->next_block_state->current_block and all_visible_according_to_vm. + * vacrel->next_block_state->current_block is set to InvalidBlockNumber if + * there are no further blocks to process. * * vacrel is an in/out parameter here; vacuum options and information about the * relation are read, members of vacrel->next_block_state are read and set as @@ -1085,12 +1141,10 @@ lazy_scan_heap(LVRelState *vacrel) * don't advance relfrozenxid when we have skipped vacuuming all-visible * blocks. * - * vmbuffer is an output parameter which, upon return, will contain the block - * from the VM containing visibility information for the next unskippable heap - * block. If we decide not to skip this heap block, the caller is responsible - * for fetching the correct VM block into vmbuffer before using it. This is - * okay as providing it as an output parameter is an optimization, not a - * requirement. + * vacrel->next_block_state->vmbuffer will contain visibility information for + * the next unskippable heap block. If we decide not to skip this heap block, + * the caller is responsible for fetching the correct VM block into the + * vmbuffer before using it. * * Note: our opinion of which blocks can be skipped can go stale immediately. * It's okay if caller "misses" a page whose all-visible or all-frozen marking @@ -1100,9 +1154,9 @@ lazy_scan_heap(LVRelState *vacrel) * older XIDs/MXIDs. The vacrel->skippedallvis flag will be set here when the * choice to skip such a range is actually made, making everything safe.) */ -static bool -heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, - BlockNumber *blkno, bool *all_visible_according_to_vm) +static void +heap_vac_scan_next_block(LVRelState *vacrel, + bool *all_visible_according_to_vm) { /* Relies on InvalidBlockNumber + 1 == 0 */ BlockNumber next_block = vacrel->next_block_state.current_block + 1; @@ -1129,8 +1183,8 @@ heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, */ if (next_block >= vacrel->rel_pages) { - vacrel->next_block_state.current_block = *blkno = InvalidBlockNumber; - return false; + vacrel->next_block_state.current_block = InvalidBlockNumber; + return; } if (vacrel->next_block_state.next_unskippable_block == InvalidBlockNumber || @@ -1144,7 +1198,7 @@ heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, { uint8 mapbits = visibilitymap_get_status(vacrel->rel, next_unskippable_block, - vmbuffer); + &vacrel->next_block_state.next_unskippable_vmbuffer); vacrel->next_block_state.next_unskippable_allvis = mapbits & VISIBILITYMAP_ALL_VISIBLE; @@ -1224,8 +1278,7 @@ heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, else *all_visible_according_to_vm = true; - vacrel->next_block_state.current_block = *blkno = next_block; - return true; + vacrel->next_block_state.current_block = next_block; } /* -- 2.40.1 --3uc3jkcn3zcvmfik Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v7-0007-Vacuum-second-pass-uses-Streaming-Read-interface.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH v7 6/7] Vacuum first pass uses Streaming Read interface @ 2023-12-31 16:29 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Melanie Plageman @ 2023-12-31 16:29 UTC (permalink / raw) Now vacuum's first pass, which HOT prunes and records the TIDs of non-removable dead tuples, uses the streaming read API by implementing a streaming read callback which invokes heap_vac_scan_next_block(). --- src/backend/access/heap/vacuumlazy.c | 131 +++++++++++++++++++-------- 1 file changed, 92 insertions(+), 39 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index d2c8f27fc57..d07a2a58b15 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -54,6 +54,7 @@ #include "storage/bufmgr.h" #include "storage/freespace.h" #include "storage/lmgr.h" +#include "storage/streaming_read.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" @@ -168,7 +169,12 @@ typedef struct LVRelState char *relnamespace; char *relname; char *indname; /* Current index name */ - BlockNumber blkno; /* used only for heap operations */ + + /* + * The current block being processed by vacuum. Used only for heap + * operations. Primarily for error reporting and logging. + */ + BlockNumber blkno; OffsetNumber offnum; /* used only for heap operations */ VacErrPhase phase; bool verbose; /* VACUUM VERBOSE? */ @@ -220,6 +226,12 @@ typedef struct LVRelState BlockNumber next_unskippable_block; /* Next unskippable block's visibility status */ bool next_unskippable_allvis; + + /* + * Buffer containing block of VM with visibility information for + * next_unskippable_block. + */ + Buffer next_unskippable_vmbuffer; } next_block_state; } LVRelState; @@ -233,8 +245,7 @@ typedef struct LVSavedErrInfo /* non-export function prototypes */ static void lazy_scan_heap(LVRelState *vacrel); -static bool heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, - BlockNumber *blkno, +static void heap_vac_scan_next_block(LVRelState *vacrel, bool *all_visible_according_to_vm); static bool lazy_scan_new_or_empty(LVRelState *vacrel, Buffer buf, BlockNumber blkno, Page page, @@ -777,6 +788,47 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } +static BlockNumber +vacuum_scan_pgsr_next(PgStreamingRead *pgsr, + void *pgsr_private, void *per_buffer_data) +{ + LVRelState *vacrel = pgsr_private; + bool *all_visible_according_to_vm = per_buffer_data; + + heap_vac_scan_next_block(vacrel, + all_visible_according_to_vm); + + /* + * If there are no further blocks to vacuum in the relation, release the + * vmbuffer. + */ + if (!BlockNumberIsValid(vacrel->next_block_state.current_block) && + BufferIsValid(vacrel->next_block_state.next_unskippable_vmbuffer)) + { + ReleaseBuffer(vacrel->next_block_state.next_unskippable_vmbuffer); + vacrel->next_block_state.next_unskippable_vmbuffer = InvalidBuffer; + } + + return vacrel->next_block_state.current_block; +} + +static inline PgStreamingRead * +vac_scan_pgsr_alloc(LVRelState *vacrel, PgStreamingReadBufferCB next_block_cb) +{ + PgStreamingRead *result = pg_streaming_read_buffer_alloc(PGSR_FLAG_MAINTENANCE, vacrel, + sizeof(bool), vacrel->bstrategy, BMR_REL(vacrel->rel), + MAIN_FORKNUM, next_block_cb); + + /* + * Initialize for first heap_vac_scan_next_block() call. These rely on + * InvalidBlockNumber + 1 = 0 + */ + vacrel->next_block_state.current_block = InvalidBlockNumber; + vacrel->next_block_state.next_unskippable_block = InvalidBlockNumber; + + return result; +} + /* * lazy_scan_heap() -- workhorse function for VACUUM * @@ -816,10 +868,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, static void lazy_scan_heap(LVRelState *vacrel) { + Buffer buf; BlockNumber rel_pages = vacrel->rel_pages, - blkno, next_fsm_block_to_vacuum = 0; - bool all_visible_according_to_vm; + bool *all_visible_according_to_vm; VacDeadItems *dead_items = vacrel->dead_items; Buffer vmbuffer = InvalidBuffer; @@ -830,23 +882,27 @@ lazy_scan_heap(LVRelState *vacrel) }; int64 initprog_val[3]; + PgStreamingRead *pgsr = vac_scan_pgsr_alloc(vacrel, vacuum_scan_pgsr_next); + /* Report that we're scanning the heap, advertising total # of blocks */ initprog_val[0] = PROGRESS_VACUUM_PHASE_SCAN_HEAP; initprog_val[1] = rel_pages; initprog_val[2] = dead_items->max_items; pgstat_progress_update_multi_param(3, initprog_index, initprog_val); - /* Initialize for first heap_vac_scan_next_block() call */ - vacrel->next_block_state.current_block = InvalidBlockNumber; - vacrel->next_block_state.next_unskippable_block = InvalidBlockNumber; - - while (heap_vac_scan_next_block(vacrel, &vmbuffer, - &blkno, &all_visible_according_to_vm)) + while (BufferIsValid(buf = pg_streaming_read_buffer_get_next(pgsr, + (void **) &all_visible_according_to_vm))) { - Buffer buf; Page page; bool has_lpdead_items; bool got_cleanup_lock = false; + BlockNumber blkno; + + vacrel->blkno = blkno = BufferGetBlockNumber(buf); + + CheckBufferIsPinnedOnce(buf); + + page = BufferGetPage(buf); vacrel->scanned_pages++; @@ -914,9 +970,6 @@ lazy_scan_heap(LVRelState *vacrel) */ visibilitymap_pin(vacrel->rel, blkno, &vmbuffer); - buf = ReadBufferExtended(vacrel->rel, MAIN_FORKNUM, blkno, RBM_NORMAL, - vacrel->bstrategy); - page = BufferGetPage(buf); /* * We need a buffer cleanup lock to prune HOT chains and defragment @@ -973,7 +1026,7 @@ lazy_scan_heap(LVRelState *vacrel) */ if (got_cleanup_lock) lazy_scan_prune(vacrel, buf, blkno, page, - vmbuffer, all_visible_according_to_vm, + vmbuffer, *all_visible_according_to_vm, &has_lpdead_items); /* @@ -1030,7 +1083,7 @@ lazy_scan_heap(LVRelState *vacrel) } /* report that everything is now scanned */ - pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, vacrel->rel_pages); /* now we can compute the new value for pg_class.reltuples */ vacrel->new_live_tuples = vac_estimate_reltuples(vacrel->rel, rel_pages, @@ -1045,6 +1098,8 @@ lazy_scan_heap(LVRelState *vacrel) Max(vacrel->new_live_tuples, 0) + vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; + pg_streaming_read_free(pgsr); + /* * Do index vacuuming (call each index's ambulkdelete routine), then do * related heap vacuuming @@ -1056,11 +1111,11 @@ lazy_scan_heap(LVRelState *vacrel) * Vacuum the remainder of the Free Space Map. We must do this whether or * not there were indexes, and whether or not we bypassed index vacuuming. */ - if (blkno > next_fsm_block_to_vacuum) - FreeSpaceMapVacuumRange(vacrel->rel, next_fsm_block_to_vacuum, blkno); + if (vacrel->rel_pages > next_fsm_block_to_vacuum) + FreeSpaceMapVacuumRange(vacrel->rel, next_fsm_block_to_vacuum, vacrel->rel_pages); /* report all blocks vacuumed */ - pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, vacrel->rel_pages); /* Do final index cleanup (call each index's amvacuumcleanup routine) */ if (vacrel->nindexes > 0 && vacrel->do_index_cleanup) @@ -1072,12 +1127,13 @@ lazy_scan_heap(LVRelState *vacrel) * * lazy_scan_heap() calls here every time it needs to get the next block to * prune and vacuum, using the visibility map, vacuum options, and various - * thresholds to skip blocks which do not need to be processed and set blkno to - * the next block that actually needs to be processed. + * thresholds to skip blocks which do not need to be processed and set + * current_block to the next block that actually needs to be processed. * - * The block number and visibility status of the next block to process are set - * in blkno and all_visible_according_to_vm. heap_vac_scan_next_block() - * returns false if there are no further blocks to process. + * The number and visibility status of the next block to process are set in + * vacrel->next_block_state->current_block and all_visible_according_to_vm. + * vacrel->next_block_state->current_block is set to InvalidBlockNumber if + * there are no further blocks to process. * * vacrel is an in/out parameter here; vacuum options and information about the * relation are read, members of vacrel->next_block_state are read and set as @@ -1085,12 +1141,10 @@ lazy_scan_heap(LVRelState *vacrel) * don't advance relfrozenxid when we have skipped vacuuming all-visible * blocks. * - * vmbuffer is an output parameter which, upon return, will contain the block - * from the VM containing visibility information for the next unskippable heap - * block. If we decide not to skip this heap block, the caller is responsible - * for fetching the correct VM block into vmbuffer before using it. This is - * okay as providing it as an output parameter is an optimization, not a - * requirement. + * vacrel->next_block_state->vmbuffer will contain visibility information for + * the next unskippable heap block. If we decide not to skip this heap block, + * the caller is responsible for fetching the correct VM block into the + * vmbuffer before using it. * * Note: our opinion of which blocks can be skipped can go stale immediately. * It's okay if caller "misses" a page whose all-visible or all-frozen marking @@ -1100,9 +1154,9 @@ lazy_scan_heap(LVRelState *vacrel) * older XIDs/MXIDs. The vacrel->skippedallvis flag will be set here when the * choice to skip such a range is actually made, making everything safe.) */ -static bool -heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, - BlockNumber *blkno, bool *all_visible_according_to_vm) +static void +heap_vac_scan_next_block(LVRelState *vacrel, + bool *all_visible_according_to_vm) { /* Relies on InvalidBlockNumber + 1 == 0 */ BlockNumber next_block = vacrel->next_block_state.current_block + 1; @@ -1129,8 +1183,8 @@ heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, */ if (next_block >= vacrel->rel_pages) { - vacrel->next_block_state.current_block = *blkno = InvalidBlockNumber; - return false; + vacrel->next_block_state.current_block = InvalidBlockNumber; + return; } if (vacrel->next_block_state.next_unskippable_block == InvalidBlockNumber || @@ -1144,7 +1198,7 @@ heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, { uint8 mapbits = visibilitymap_get_status(vacrel->rel, next_unskippable_block, - vmbuffer); + &vacrel->next_block_state.next_unskippable_vmbuffer); vacrel->next_block_state.next_unskippable_allvis = mapbits & VISIBILITYMAP_ALL_VISIBLE; @@ -1224,8 +1278,7 @@ heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, else *all_visible_according_to_vm = true; - vacrel->next_block_state.current_block = *blkno = next_block; - return true; + vacrel->next_block_state.current_block = next_block; } /* -- 2.40.1 --3uc3jkcn3zcvmfik Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v7-0007-Vacuum-second-pass-uses-Streaming-Read-interface.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH v7 6/7] Vacuum first pass uses Streaming Read interface @ 2023-12-31 16:29 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Melanie Plageman @ 2023-12-31 16:29 UTC (permalink / raw) Now vacuum's first pass, which HOT prunes and records the TIDs of non-removable dead tuples, uses the streaming read API by implementing a streaming read callback which invokes heap_vac_scan_next_block(). --- src/backend/access/heap/vacuumlazy.c | 131 +++++++++++++++++++-------- 1 file changed, 92 insertions(+), 39 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index d2c8f27fc57..d07a2a58b15 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -54,6 +54,7 @@ #include "storage/bufmgr.h" #include "storage/freespace.h" #include "storage/lmgr.h" +#include "storage/streaming_read.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" @@ -168,7 +169,12 @@ typedef struct LVRelState char *relnamespace; char *relname; char *indname; /* Current index name */ - BlockNumber blkno; /* used only for heap operations */ + + /* + * The current block being processed by vacuum. Used only for heap + * operations. Primarily for error reporting and logging. + */ + BlockNumber blkno; OffsetNumber offnum; /* used only for heap operations */ VacErrPhase phase; bool verbose; /* VACUUM VERBOSE? */ @@ -220,6 +226,12 @@ typedef struct LVRelState BlockNumber next_unskippable_block; /* Next unskippable block's visibility status */ bool next_unskippable_allvis; + + /* + * Buffer containing block of VM with visibility information for + * next_unskippable_block. + */ + Buffer next_unskippable_vmbuffer; } next_block_state; } LVRelState; @@ -233,8 +245,7 @@ typedef struct LVSavedErrInfo /* non-export function prototypes */ static void lazy_scan_heap(LVRelState *vacrel); -static bool heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, - BlockNumber *blkno, +static void heap_vac_scan_next_block(LVRelState *vacrel, bool *all_visible_according_to_vm); static bool lazy_scan_new_or_empty(LVRelState *vacrel, Buffer buf, BlockNumber blkno, Page page, @@ -777,6 +788,47 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } +static BlockNumber +vacuum_scan_pgsr_next(PgStreamingRead *pgsr, + void *pgsr_private, void *per_buffer_data) +{ + LVRelState *vacrel = pgsr_private; + bool *all_visible_according_to_vm = per_buffer_data; + + heap_vac_scan_next_block(vacrel, + all_visible_according_to_vm); + + /* + * If there are no further blocks to vacuum in the relation, release the + * vmbuffer. + */ + if (!BlockNumberIsValid(vacrel->next_block_state.current_block) && + BufferIsValid(vacrel->next_block_state.next_unskippable_vmbuffer)) + { + ReleaseBuffer(vacrel->next_block_state.next_unskippable_vmbuffer); + vacrel->next_block_state.next_unskippable_vmbuffer = InvalidBuffer; + } + + return vacrel->next_block_state.current_block; +} + +static inline PgStreamingRead * +vac_scan_pgsr_alloc(LVRelState *vacrel, PgStreamingReadBufferCB next_block_cb) +{ + PgStreamingRead *result = pg_streaming_read_buffer_alloc(PGSR_FLAG_MAINTENANCE, vacrel, + sizeof(bool), vacrel->bstrategy, BMR_REL(vacrel->rel), + MAIN_FORKNUM, next_block_cb); + + /* + * Initialize for first heap_vac_scan_next_block() call. These rely on + * InvalidBlockNumber + 1 = 0 + */ + vacrel->next_block_state.current_block = InvalidBlockNumber; + vacrel->next_block_state.next_unskippable_block = InvalidBlockNumber; + + return result; +} + /* * lazy_scan_heap() -- workhorse function for VACUUM * @@ -816,10 +868,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, static void lazy_scan_heap(LVRelState *vacrel) { + Buffer buf; BlockNumber rel_pages = vacrel->rel_pages, - blkno, next_fsm_block_to_vacuum = 0; - bool all_visible_according_to_vm; + bool *all_visible_according_to_vm; VacDeadItems *dead_items = vacrel->dead_items; Buffer vmbuffer = InvalidBuffer; @@ -830,23 +882,27 @@ lazy_scan_heap(LVRelState *vacrel) }; int64 initprog_val[3]; + PgStreamingRead *pgsr = vac_scan_pgsr_alloc(vacrel, vacuum_scan_pgsr_next); + /* Report that we're scanning the heap, advertising total # of blocks */ initprog_val[0] = PROGRESS_VACUUM_PHASE_SCAN_HEAP; initprog_val[1] = rel_pages; initprog_val[2] = dead_items->max_items; pgstat_progress_update_multi_param(3, initprog_index, initprog_val); - /* Initialize for first heap_vac_scan_next_block() call */ - vacrel->next_block_state.current_block = InvalidBlockNumber; - vacrel->next_block_state.next_unskippable_block = InvalidBlockNumber; - - while (heap_vac_scan_next_block(vacrel, &vmbuffer, - &blkno, &all_visible_according_to_vm)) + while (BufferIsValid(buf = pg_streaming_read_buffer_get_next(pgsr, + (void **) &all_visible_according_to_vm))) { - Buffer buf; Page page; bool has_lpdead_items; bool got_cleanup_lock = false; + BlockNumber blkno; + + vacrel->blkno = blkno = BufferGetBlockNumber(buf); + + CheckBufferIsPinnedOnce(buf); + + page = BufferGetPage(buf); vacrel->scanned_pages++; @@ -914,9 +970,6 @@ lazy_scan_heap(LVRelState *vacrel) */ visibilitymap_pin(vacrel->rel, blkno, &vmbuffer); - buf = ReadBufferExtended(vacrel->rel, MAIN_FORKNUM, blkno, RBM_NORMAL, - vacrel->bstrategy); - page = BufferGetPage(buf); /* * We need a buffer cleanup lock to prune HOT chains and defragment @@ -973,7 +1026,7 @@ lazy_scan_heap(LVRelState *vacrel) */ if (got_cleanup_lock) lazy_scan_prune(vacrel, buf, blkno, page, - vmbuffer, all_visible_according_to_vm, + vmbuffer, *all_visible_according_to_vm, &has_lpdead_items); /* @@ -1030,7 +1083,7 @@ lazy_scan_heap(LVRelState *vacrel) } /* report that everything is now scanned */ - pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, vacrel->rel_pages); /* now we can compute the new value for pg_class.reltuples */ vacrel->new_live_tuples = vac_estimate_reltuples(vacrel->rel, rel_pages, @@ -1045,6 +1098,8 @@ lazy_scan_heap(LVRelState *vacrel) Max(vacrel->new_live_tuples, 0) + vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; + pg_streaming_read_free(pgsr); + /* * Do index vacuuming (call each index's ambulkdelete routine), then do * related heap vacuuming @@ -1056,11 +1111,11 @@ lazy_scan_heap(LVRelState *vacrel) * Vacuum the remainder of the Free Space Map. We must do this whether or * not there were indexes, and whether or not we bypassed index vacuuming. */ - if (blkno > next_fsm_block_to_vacuum) - FreeSpaceMapVacuumRange(vacrel->rel, next_fsm_block_to_vacuum, blkno); + if (vacrel->rel_pages > next_fsm_block_to_vacuum) + FreeSpaceMapVacuumRange(vacrel->rel, next_fsm_block_to_vacuum, vacrel->rel_pages); /* report all blocks vacuumed */ - pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, vacrel->rel_pages); /* Do final index cleanup (call each index's amvacuumcleanup routine) */ if (vacrel->nindexes > 0 && vacrel->do_index_cleanup) @@ -1072,12 +1127,13 @@ lazy_scan_heap(LVRelState *vacrel) * * lazy_scan_heap() calls here every time it needs to get the next block to * prune and vacuum, using the visibility map, vacuum options, and various - * thresholds to skip blocks which do not need to be processed and set blkno to - * the next block that actually needs to be processed. + * thresholds to skip blocks which do not need to be processed and set + * current_block to the next block that actually needs to be processed. * - * The block number and visibility status of the next block to process are set - * in blkno and all_visible_according_to_vm. heap_vac_scan_next_block() - * returns false if there are no further blocks to process. + * The number and visibility status of the next block to process are set in + * vacrel->next_block_state->current_block and all_visible_according_to_vm. + * vacrel->next_block_state->current_block is set to InvalidBlockNumber if + * there are no further blocks to process. * * vacrel is an in/out parameter here; vacuum options and information about the * relation are read, members of vacrel->next_block_state are read and set as @@ -1085,12 +1141,10 @@ lazy_scan_heap(LVRelState *vacrel) * don't advance relfrozenxid when we have skipped vacuuming all-visible * blocks. * - * vmbuffer is an output parameter which, upon return, will contain the block - * from the VM containing visibility information for the next unskippable heap - * block. If we decide not to skip this heap block, the caller is responsible - * for fetching the correct VM block into vmbuffer before using it. This is - * okay as providing it as an output parameter is an optimization, not a - * requirement. + * vacrel->next_block_state->vmbuffer will contain visibility information for + * the next unskippable heap block. If we decide not to skip this heap block, + * the caller is responsible for fetching the correct VM block into the + * vmbuffer before using it. * * Note: our opinion of which blocks can be skipped can go stale immediately. * It's okay if caller "misses" a page whose all-visible or all-frozen marking @@ -1100,9 +1154,9 @@ lazy_scan_heap(LVRelState *vacrel) * older XIDs/MXIDs. The vacrel->skippedallvis flag will be set here when the * choice to skip such a range is actually made, making everything safe.) */ -static bool -heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, - BlockNumber *blkno, bool *all_visible_according_to_vm) +static void +heap_vac_scan_next_block(LVRelState *vacrel, + bool *all_visible_according_to_vm) { /* Relies on InvalidBlockNumber + 1 == 0 */ BlockNumber next_block = vacrel->next_block_state.current_block + 1; @@ -1129,8 +1183,8 @@ heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, */ if (next_block >= vacrel->rel_pages) { - vacrel->next_block_state.current_block = *blkno = InvalidBlockNumber; - return false; + vacrel->next_block_state.current_block = InvalidBlockNumber; + return; } if (vacrel->next_block_state.next_unskippable_block == InvalidBlockNumber || @@ -1144,7 +1198,7 @@ heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, { uint8 mapbits = visibilitymap_get_status(vacrel->rel, next_unskippable_block, - vmbuffer); + &vacrel->next_block_state.next_unskippable_vmbuffer); vacrel->next_block_state.next_unskippable_allvis = mapbits & VISIBILITYMAP_ALL_VISIBLE; @@ -1224,8 +1278,7 @@ heap_vac_scan_next_block(LVRelState *vacrel, Buffer *vmbuffer, else *all_visible_according_to_vm = true; - vacrel->next_block_state.current_block = *blkno = next_block; - return true; + vacrel->next_block_state.current_block = next_block; } /* -- 2.40.1 --3uc3jkcn3zcvmfik Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v7-0007-Vacuum-second-pass-uses-Streaming-Read-interface.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH v5a 6/7] Vacuum first pass uses Streaming Read interface @ 2023-12-31 16:29 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Melanie Plageman @ 2023-12-31 16:29 UTC (permalink / raw) Now vacuum's first pass, which HOT prunes and records the TIDs of non-removable dead tuples, uses the streaming read API by implementing a streaming read callback which invokes heap_vac_scan_get_next_block(). --- src/backend/access/heap/vacuumlazy.c | 79 +++++++++++++++++++++------- 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 65d257aab83..fbbc87938e4 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -54,6 +54,7 @@ #include "storage/bufmgr.h" #include "storage/freespace.h" #include "storage/lmgr.h" +#include "storage/streaming_read.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" @@ -168,7 +169,12 @@ typedef struct LVRelState char *relnamespace; char *relname; char *indname; /* Current index name */ - BlockNumber blkno; /* used only for heap operations */ + + /* + * The current block being processed by vacuum. Used only for heap + * operations. Primarily for error reporting and logging. + */ + BlockNumber blkno; OffsetNumber offnum; /* used only for heap operations */ VacErrPhase phase; bool verbose; /* VACUUM VERBOSE? */ @@ -189,6 +195,12 @@ typedef struct LVRelState BlockNumber missed_dead_pages; /* # pages with missed dead tuples */ BlockNumber nonempty_pages; /* actually, last nonempty page + 1 */ + /* + * The most recent block submitted in the streaming read callback by the + * first vacuum pass. + */ + BlockNumber blkno_prefetch; + /* Statistics output by us, for table */ double new_rel_tuples; /* new estimated total # of tuples */ double new_live_tuples; /* new estimated total # of live tuples */ @@ -232,7 +244,7 @@ typedef struct LVSavedErrInfo /* non-export function prototypes */ static void lazy_scan_heap(LVRelState *vacrel); -static bool heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, +static void heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, BlockNumber *blkno, bool *all_visible_according_to_vm); static bool lazy_scan_new_or_empty(LVRelState *vacrel, Buffer buf, @@ -416,6 +428,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->nonempty_pages = 0; /* dead_items_alloc allocates vacrel->dead_items later on */ + /* relies on InvalidBlockNumber overflowing to 0 */ + vacrel->blkno_prefetch = InvalidBlockNumber; + /* Allocate/initialize output statistics state */ vacrel->new_rel_tuples = 0; vacrel->new_live_tuples = 0; @@ -776,6 +791,22 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, } } +static BlockNumber +vacuum_scan_pgsr_next(PgStreamingRead *pgsr, + void *pgsr_private, void *per_buffer_data) +{ + LVRelState *vacrel = pgsr_private; + bool *all_visible_according_to_vm = per_buffer_data; + + vacrel->blkno_prefetch++; + + heap_vac_scan_get_next_block(vacrel, + vacrel->blkno_prefetch, &vacrel->blkno_prefetch, + all_visible_according_to_vm); + + return vacrel->blkno_prefetch; +} + /* * lazy_scan_heap() -- workhorse function for VACUUM * @@ -815,12 +846,11 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, static void lazy_scan_heap(LVRelState *vacrel) { + Buffer buf; BlockNumber rel_pages = vacrel->rel_pages, next_fsm_block_to_vacuum = 0; - bool all_visible_according_to_vm; + bool *all_visible_according_to_vm; - /* relies on InvalidBlockNumber overflowing to 0 */ - BlockNumber blkno = InvalidBlockNumber; VacDeadItems *dead_items = vacrel->dead_items; const int initprog_index[] = { PROGRESS_VACUUM_PHASE, @@ -828,6 +858,11 @@ lazy_scan_heap(LVRelState *vacrel) PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + PgStreamingRead *pgsr; + + pgsr = pg_streaming_read_buffer_alloc(PGSR_FLAG_MAINTENANCE, vacrel, + sizeof(bool), vacrel->bstrategy, BMR_REL(vacrel->rel), + MAIN_FORKNUM, vacuum_scan_pgsr_next); /* Report that we're scanning the heap, advertising total # of blocks */ initprog_val[0] = PROGRESS_VACUUM_PHASE_SCAN_HEAP; @@ -838,13 +873,19 @@ lazy_scan_heap(LVRelState *vacrel) vacrel->skip.next_unskippable_block = InvalidBlockNumber; vacrel->skip.vmbuffer = InvalidBuffer; - while (heap_vac_scan_get_next_block(vacrel, blkno + 1, - &blkno, &all_visible_according_to_vm)) + while (BufferIsValid(buf = + pg_streaming_read_buffer_get_next(pgsr, (void **) &all_visible_according_to_vm))) { - Buffer buf; Page page; bool has_lpdead_items; bool got_cleanup_lock = false; + BlockNumber blkno; + + vacrel->blkno = blkno = BufferGetBlockNumber(buf); + + CheckBufferIsPinnedOnce(buf); + + page = BufferGetPage(buf); vacrel->scanned_pages++; @@ -912,9 +953,6 @@ lazy_scan_heap(LVRelState *vacrel) */ visibilitymap_pin(vacrel->rel, blkno, &vacrel->skip.vmbuffer); - buf = ReadBufferExtended(vacrel->rel, MAIN_FORKNUM, blkno, RBM_NORMAL, - vacrel->bstrategy); - page = BufferGetPage(buf); /* * We need a buffer cleanup lock to prune HOT chains and defragment @@ -970,7 +1008,7 @@ lazy_scan_heap(LVRelState *vacrel) */ if (got_cleanup_lock) lazy_scan_prune(vacrel, buf, blkno, page, - all_visible_according_to_vm, + *all_visible_according_to_vm, &has_lpdead_items); /* @@ -1027,7 +1065,7 @@ lazy_scan_heap(LVRelState *vacrel) } /* report that everything is now scanned */ - pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, vacrel->rel_pages); /* now we can compute the new value for pg_class.reltuples */ vacrel->new_live_tuples = vac_estimate_reltuples(vacrel->rel, rel_pages, @@ -1042,6 +1080,8 @@ lazy_scan_heap(LVRelState *vacrel) Max(vacrel->new_live_tuples, 0) + vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; + pg_streaming_read_free(pgsr); + /* * Do index vacuuming (call each index's ambulkdelete routine), then do * related heap vacuuming @@ -1053,11 +1093,11 @@ lazy_scan_heap(LVRelState *vacrel) * Vacuum the remainder of the Free Space Map. We must do this whether or * not there were indexes, and whether or not we bypassed index vacuuming. */ - if (blkno > next_fsm_block_to_vacuum) - FreeSpaceMapVacuumRange(vacrel->rel, next_fsm_block_to_vacuum, blkno); + if (vacrel->rel_pages > next_fsm_block_to_vacuum) + FreeSpaceMapVacuumRange(vacrel->rel, next_fsm_block_to_vacuum, vacrel->rel_pages); /* report all blocks vacuumed */ - pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, vacrel->rel_pages); /* Do final index cleanup (call each index's amvacuumcleanup routine) */ if (vacrel->nindexes > 0 && vacrel->do_index_cleanup) @@ -1090,7 +1130,7 @@ lazy_scan_heap(LVRelState *vacrel) * * The block number and visibility status of the next block to process are set * in blkno and all_visible_according_to_vm. heap_vac_scan_get_next_block() - * returns false if there are no further blocks to process. + * sets blkno to InvalidBlockNumber if there are no further blocks to process. * * vacrel is an in/out parameter here; vacuum options and information about the * relation are read and vacrel->skippedallvis is set to ensure we don't @@ -1110,7 +1150,7 @@ lazy_scan_heap(LVRelState *vacrel) * older XIDs/MXIDs. The vacrel->skippedallvis flag will be set here when the * choice to skip such a range is actually made, making everything safe.) */ -static bool +static void heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, BlockNumber *blkno, bool *all_visible_according_to_vm) { @@ -1119,7 +1159,7 @@ heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, if (next_block >= vacrel->rel_pages) { *blkno = InvalidBlockNumber; - return false; + return; } if (vacrel->skip.next_unskippable_block == InvalidBlockNumber || @@ -1214,7 +1254,6 @@ heap_vac_scan_get_next_block(LVRelState *vacrel, BlockNumber next_block, *all_visible_according_to_vm = true; *blkno = next_block; - return true; } /* -- 2.40.1 --lb5e4aqx4b6ik5lq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5a-0007-Vacuum-second-pass-uses-Streaming-Read-interface.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: post-freeze damage control @ 2024-04-12 14:54 Alexander Korotkov <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Alexander Korotkov @ 2024-04-12 14:54 UTC (permalink / raw) To: Robert Haas <[email protected]>; +Cc: Tom Lane <[email protected]>; pgsql-hackers On Wed, Apr 10, 2024 at 5:27 PM Robert Haas <[email protected]> wrote: > > On Mon, Apr 8, 2024 at 10:12 PM Tom Lane <[email protected]> wrote: > > I have another one that I'm not terribly happy about: > > > > Author: Alexander Korotkov <[email protected]> > > Branch: master [72bd38cc9] 2024-04-08 01:27:52 +0300 > > > > Transform OR clauses to ANY expression > > I realize that this has been reverted now, but what's really > frustrating about this case is that I reviewed this patch before and > gave feedback similar to some of the feedback you gave, and it just > didn't matter, and the patch was committed anyway. > > > I don't know that I'd call it scary exactly, but I do think it > > was premature. A week ago there was no consensus that it was > > ready to commit, but Alexander pushed it (or half of it, anyway) > > despite that. A few concrete concerns: > > > > * Yet another planner GUC. Do we really need or want that? > > IMHO, no, and I said so in > https://www.postgresql.org/message-id/CA%2BTgmob%3DebuCHFSw327b55DJzE3JtOuZ5owxob%2BMgErb4me_Ag%40ma... > > > * What the medical community would call off-label usage of > > query jumbling. I'm not sure this is even correct as-used, > > and for sure it's using that code for something never intended. > > Nor is the added code adequately (as in, at all) documented. > > And I raised this point here: > https://www.postgresql.org/message-id/CA%2BTgmoZCgP6FrBQEusn4yaWm02XU8OPeoEMk91q7PRBgwaAkFw%40mail.g... > > > * Patch refuses to group anything but Consts into the SAOP > > transformation. I realize that if you want to produce an > > array Const you need Const inputs, but I wonder why it > > wasn't considered to produce an ARRAY[] construct if there > > are available clauses with pseudo-constant (eg Param) > > comparison values. > > > > * I really, really dislike jamming this logic into prepqual.c, > > where it has no business being. I note that it was shoved > > into process_duplicate_ors without even the courtesy of > > expanding the header comment: > > > > * process_duplicate_ors > > * Given a list of exprs which are ORed together, try to apply > > * the inverse OR distributive law. > > > > Another reason to think this wasn't a very well chosen place is > > that the file's list of #include's went from 4 entries to 11. > > Somebody should have twigged to the idea that this was off-topic > > for prepqual.c. > > All of this seems like it might be related to my comments in the above > email about the transformation being done too early. > > > * OrClauseGroupKey is not a Node type, so why does it have > > a NodeTag? I wonder what value will appear in that field, > > and what will happen if the struct is passed to any code > > that expects real Nodes. > > I don't think I raised this issue. > > > I could probably find some other nits if I spent more time > > on it, but I think these are sufficient to show that this > > was not commit-ready. > > Just imagine if someone had taken time to give similar feedback before > the commit. FWIW, I made my conclusion that it isn't worth to commit stuff like this without explicit consent from Tom. As well as it isn't worth to commit table AM changes without explicit consent from Andres. And it isn't worth it to postpone large features to the last CF (it's better to postpone to the next release then). ------ Regards, Alexander Korotkov ^ permalink raw reply [nested|flat] 7+ messages in thread
end of thread, other threads:[~2024-04-12 14:54 UTC | newest] Thread overview: 7+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2023-12-31 16:29 [PATCH v5a 6/7] Vacuum first pass uses Streaming Read interface Melanie Plageman <[email protected]> 2023-12-31 16:29 [PATCH v7 6/7] Vacuum first pass uses Streaming Read interface Melanie Plageman <[email protected]> 2023-12-31 16:29 [PATCH v7 6/7] Vacuum first pass uses Streaming Read interface Melanie Plageman <[email protected]> 2023-12-31 16:29 [PATCH v5a 6/7] Vacuum first pass uses Streaming Read interface Melanie Plageman <[email protected]> 2023-12-31 16:29 [PATCH v7 6/7] Vacuum first pass uses Streaming Read interface Melanie Plageman <[email protected]> 2023-12-31 16:29 [PATCH v5a 6/7] Vacuum first pass uses Streaming Read interface Melanie Plageman <[email protected]> 2024-04-12 14:54 Re: post-freeze damage control Alexander Korotkov <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox