agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v3] vacuum errcontext to show block being processed 30+ messages / 2 participants [nested] [flat]
* [PATCH v3] vacuum errcontext to show block being processed @ 2019-11-20 20:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-11-20 20:53 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 043ebb4..9376989 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -138,6 +138,12 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relname; + char *relnamespace; + BlockNumber blkno; +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -175,6 +181,7 @@ static bool lazy_tid_reaped(ItemPointer itemptr, void *state); static int vac_cmp_itemptr(const void *left, const void *right); static bool heap_page_is_all_visible(Relation rel, Buffer buf, TransactionId *visibility_cutoff_xid, bool *all_frozen); +static void vacuum_error_callback(void *arg); /* @@ -524,6 +531,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg cbarg; pg_rusage_init(&ru0); @@ -635,6 +644,15 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + errcallback.callback = vacuum_error_callback; + cbarg.relname = relname; + cbarg.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + cbarg.blkno = 0; /* Not known yet */ + errcallback.arg = (void *) &cbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -658,6 +676,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); + cbarg.blkno = blkno; + if (blkno == next_unskippable_block) { /* Time to advance next_unskippable_block */ @@ -817,7 +837,6 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, buf = ReadBufferExtended(onerel, MAIN_FORKNUM, blkno, RBM_NORMAL, vac_strategy); - /* We need buffer cleanup lock so that we can prune HOT chains. */ if (!ConditionalLockBufferForCleanup(buf)) { @@ -1388,6 +1407,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -2354,3 +2376,15 @@ heap_page_is_all_visible(Relation rel, Buffer buf, return all_visible; } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); +} -- 2.7.4 --xaMk4Io5JJdpkLEb-- ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v1] vacuum errcontext to show block being processed @ 2019-11-20 20:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-11-20 20:53 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 43 +++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index a3c4a1d..6e0938b 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -175,6 +175,7 @@ static bool lazy_tid_reaped(ItemPointer itemptr, void *state); static int vac_cmp_itemptr(const void *left, const void *right); static bool heap_page_is_all_visible(Relation rel, Buffer buf, TransactionId *visibility_cutoff_xid, bool *all_frozen); +static void vacuum_error_callback(void *arg); /* @@ -517,13 +518,14 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, BlockNumber next_unskippable_block; bool skipping_blocks; xl_heap_freeze_tuple *frozen; - StringInfoData buf; + StringInfoData sbuf; const int initprog_index[] = { PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_TOTAL_HEAP_BLKS, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); @@ -635,6 +637,12 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &blkno; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -1388,6 +1396,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -1481,33 +1492,33 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, * This is pretty messy, but we split it up so that we can skip emitting * individual parts of the message when not applicable. */ - initStringInfo(&buf); - appendStringInfo(&buf, + initStringInfo(&sbuf); + appendStringInfo(&sbuf, _("%.0f dead row versions cannot be removed yet, oldest xmin: %u\n"), nkeep, OldestXmin); - appendStringInfo(&buf, _("There were %.0f unused item identifiers.\n"), + appendStringInfo(&sbuf, _("There were %.0f unused item identifiers.\n"), nunused); - appendStringInfo(&buf, ngettext("Skipped %u page due to buffer pins, ", + appendStringInfo(&sbuf, ngettext("Skipped %u page due to buffer pins, ", "Skipped %u pages due to buffer pins, ", vacrelstats->pinskipped_pages), vacrelstats->pinskipped_pages); - appendStringInfo(&buf, ngettext("%u frozen page.\n", + appendStringInfo(&sbuf, ngettext("%u frozen page.\n", "%u frozen pages.\n", vacrelstats->frozenskipped_pages), vacrelstats->frozenskipped_pages); - appendStringInfo(&buf, ngettext("%u page is entirely empty.\n", + appendStringInfo(&sbuf, ngettext("%u page is entirely empty.\n", "%u pages are entirely empty.\n", empty_pages), empty_pages); - appendStringInfo(&buf, _("%s."), pg_rusage_show(&ru0)); + appendStringInfo(&sbuf, _("%s."), pg_rusage_show(&ru0)); ereport(elevel, (errmsg("\"%s\": found %.0f removable, %.0f nonremovable row versions in %u out of %u pages", RelationGetRelationName(onerel), tups_vacuumed, num_tuples, vacrelstats->scanned_pages, nblocks), - errdetail_internal("%s", buf.data))); - pfree(buf.data); + errdetail_internal("%s", sbuf.data))); + pfree(sbuf.data); } @@ -2354,3 +2365,15 @@ heap_page_is_all_visible(Relation rel, Buffer buf, return all_visible; } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + // BufferDesc *bufHdr = (BufferDesc *) arg; + Buffer *buf = (int *) arg; + + errcontext("block %u", *buf); +} -- 2.7.4 --uQr8t48UFsdbeI+V-- ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v2 1/2] vacuum errcontext to show block being processed @ 2019-11-20 20:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-11-20 20:53 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index a3c4a1d..774cad5 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -138,6 +138,11 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relname; + BlockNumber blkno; +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -175,6 +180,7 @@ static bool lazy_tid_reaped(ItemPointer itemptr, void *state); static int vac_cmp_itemptr(const void *left, const void *right); static bool heap_page_is_all_visible(Relation rel, Buffer buf, TransactionId *visibility_cutoff_xid, bool *all_frozen); +static void vacuum_error_callback(void *arg); /* @@ -524,6 +530,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg cbarg; pg_rusage_init(&ru0); @@ -635,6 +643,14 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + errcallback.callback = vacuum_error_callback; + cbarg.relname = relname; + cbarg.blkno = 0; /* Not known yet */ + errcallback.arg = (void *) &cbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -657,6 +673,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); + cbarg.blkno = blkno; if (blkno == next_unskippable_block) { @@ -817,6 +834,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, buf = ReadBufferExtended(onerel, MAIN_FORKNUM, blkno, RBM_NORMAL, vac_strategy); + // errcallback.arg = (void *) &buf; /* We need buffer cleanup lock so that we can prune HOT chains. */ if (!ConditionalLockBufferForCleanup(buf)) @@ -1388,6 +1406,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -2354,3 +2375,22 @@ heap_page_is_all_visible(Relation rel, Buffer buf, return all_visible; } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + errcontext("block %u of relation %s", cbarg->blkno, cbarg->relname); + // Buffer *buf = arg; + // RelFileNode rnode; + // ForkNumber forknum; + // BlockNumber blknum; + // char *path; + // BufferGetTag(*buf, &rnode, &forknum, &blknum); + // path = relpathperm(rnode, forknum); + // errcontext("block %u of relation %s", blknum, path); + // pfree(path); +} -- 2.7.4 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2-0002-Rename-buf-to-avoid-shadowing-buf-of-another-type.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v21 1/2] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) Discussion: https://www.postgresql.org/message-id/[email protected] --- src/backend/access/heap/vacuumlazy.c | 130 ++++++++++++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 1 deletion(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 03c43ef..9e69294 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -292,6 +292,14 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relnamespace; + char *relname; + char *indname; + BlockNumber blkno; /* used only for heap operations */ + int phase; /* Reusing same enums as for progress reporting */ +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -361,6 +369,9 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); +static void init_vacuum_error_callback(ErrorContextCallback *errcallback, + vacuum_error_callback_arg *errcbarg, Relation onerel, int phase); /* @@ -724,6 +735,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -870,6 +883,11 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, onerel, + PROGRESS_VACUUM_PHASE_SCAN_HEAP); + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,6 +909,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -987,6 +1007,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, vmbuffer = InvalidBuffer; } + /* Pop the error context stack while calling vacuum */ + error_context_stack = errcallback.previous; + /* Work on all the indexes, then the heap */ lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); @@ -1011,6 +1034,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, /* Report that we are once again scanning the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + + /* Set the error context while continuing heap scan */ + error_context_stack = &errcallback; } /* @@ -1597,6 +1623,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -1772,14 +1801,21 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) int npages; PGRUsage ru0; Buffer vmbuffer = InvalidBuffer; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; /* Report that we are now vacuuming the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); pg_rusage_init(&ru0); - npages = 0; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, onerel, + PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + error_context_stack = &errcallback; + + npages = 0; tupindex = 0; while (tupindex < vacrelstats->dead_tuples->num_tuples) { @@ -1791,6 +1827,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + errcbarg.blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1811,6 +1848,9 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) npages++; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (BufferIsValid(vmbuffer)) { ReleaseBuffer(vmbuffer); @@ -2318,6 +2358,8 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -2329,10 +2371,18 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, indrel, + PROGRESS_VACUUM_PHASE_VACUUM_INDEX); + error_context_stack = &errcallback; + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (IsParallelWorker()) msg = gettext_noop("scanned index \"%s\" to remove %d row versions by parallel vacuum worker"); else @@ -2359,6 +2409,8 @@ lazy_cleanup_index(Relation indrel, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + vacuum_error_callback_arg errcbarg; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); @@ -2371,8 +2423,16 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, indrel, + PROGRESS_VACUUM_PHASE_INDEX_CLEANUP); + error_context_stack = &errcallback; + *stats = index_vacuum_cleanup(&ivinfo, *stats); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (!(*stats)) return; @@ -3376,3 +3436,71 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + + switch (cbarg->phase) { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext(_("while scanning block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext(_("while vacuuming block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + errcontext(_("while vacuuming index \"%s.%s\" of relation \"%s\""), + cbarg->relnamespace, cbarg->indname, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + errcontext(_("while cleaning up index \"%s.%s\" of relation \"%s\""), + cbarg->relnamespace, cbarg->indname, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_TRUNCATE: + case PROGRESS_VACUUM_PHASE_FINAL_CLEANUP: + default: + return; /* Shouldn't happen: do nothing */ + } +} + +/* Initialize vacuum error callback */ +static void +init_vacuum_error_callback(ErrorContextCallback *errcallback, + vacuum_error_callback_arg *errcbarg, Relation rel, int phase) +{ + switch (phase) + { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + errcbarg->relname = RelationGetRelationName(rel); + errcbarg->indname = NULL; /* Not used for heap */ + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + /* rel is an index relation in index cases */ + errcbarg->relname = get_rel_name(rel->rd_index->indrelid); + errcbarg->indname = RelationGetRelationName(rel); + break; + } + + errcbarg->relnamespace = get_namespace_name(RelationGetNamespace(rel)); + errcbarg->blkno = InvalidBlockNumber; /* Not known yet */ + errcbarg->phase = phase; + + errcallback->callback = vacuum_error_callback; + errcallback->arg = errcbarg; + errcallback->previous = error_context_stack; +} -- 2.7.4 --X1bOJ3K7DJ5YkBrT Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v21-0002-add-callback-for-truncation.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v14 2/3] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 85 +++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 114428b..a62dc79 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -290,8 +290,14 @@ typedef struct LVRelStats int num_index_scans; TransactionId latestRemovedXid; bool lock_waiter_detected; -} LVRelStats; + /* Used by the error callback */ + char *relname; + char *relnamespace; + BlockNumber blkno; + char *indname; + int stage; /* 0: scan heap; 1: vacuum heap; 2: vacuum index */ +} LVRelStats; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -360,6 +366,7 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); /* @@ -721,6 +728,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); @@ -867,6 +875,17 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + vacrelstats.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats.relname = relname; + vacrelstats.blkno = InvalidBlockNumber; /* Not known yet */ + vacrelstats.stage = 0; + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -888,6 +907,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params)) + vacrelstats.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -984,12 +1005,18 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, vmbuffer = InvalidBuffer; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* Work on all the indexes, then the heap */ lazy_vacuum_all_indexes(onerel, Irel, indstats, lps, nindexes); /* Remove tuples from heap */ lazy_vacuum_heap(onerel); + /* Replace error context while continuing heap scan */ + error_context_stack = &errcallback; + /* * Forget the now-vacuumed tuples, and press on, but be careful * not to reset latestRemovedXid since we want that value to be @@ -1593,6 +1620,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -1768,11 +1798,24 @@ lazy_vacuum_heap(Relation onerel) int npages; PGRUsage ru0; Buffer vmbuffer = InvalidBuffer; + ErrorContextCallback errcallback; /* Report that we are now vacuuming the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + /* + * Setup error traceback support for ereport() + * ->relnamespace and ->relname are already set + */ + vacrelstats.blkno = InvalidBlockNumber; /* Not known yet */ + vacrelstats.stage = 1; + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + pg_rusage_init(&ru0); npages = 0; @@ -1787,6 +1830,7 @@ lazy_vacuum_heap(Relation onerel) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats.dead_tuples->itemptrs[tupindex]); + vacrelstats.blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1807,6 +1851,9 @@ lazy_vacuum_heap(Relation onerel) npages++; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (BufferIsValid(vmbuffer)) { ReleaseBuffer(vmbuffer); @@ -2314,6 +2361,8 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + ErrorContextCallback errcallback; + LVRelStats errcbarg; /* Used for error callback, only */ pg_rusage_init(&ru0); @@ -2325,10 +2374,24 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(indrel)); + errcbarg.indname = RelationGetRelationName(indrel); + errcbarg.relname = vacrelstats.relname; + errcbarg.stage = 2; + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (IsParallelWorker()) msg = gettext_noop("scanned index \"%s\" to remove %d row versions by parallel vacuum worker"); else @@ -3371,3 +3434,23 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + LVRelStats *cbarg = arg; + + if (cbarg->stage == 0) + errcontext(_("while scanning block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + else if (cbarg->stage == 1) + errcontext(_("while vacuuming block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + else if (cbarg->stage == 2) + errcontext(_("while vacuuming index \"%s\" on table \"%s.%s\""), + cbarg->indname, cbarg->relnamespace, cbarg->relname); + +} -- 2.7.4 --p2u4WfPhYOuYlOsk Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v14-0003-Avoid-extra-calls-like-GetRelationName.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v29 1/3] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) Discussion: https://www.postgresql.org/message-id/[email protected] --- src/backend/access/heap/vacuumlazy.c | 245 ++++++++++++++++++++++++--- src/tools/pgindent/typedefs.list | 1 + 2 files changed, 221 insertions(+), 25 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 03c43efc32..531f8471db 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -268,8 +268,20 @@ typedef struct LVParallelState int nindexes_parallel_condcleanup; } LVParallelState; +typedef enum +{ + VACUUM_ERRCB_PHASE_UNKNOWN, + VACUUM_ERRCB_PHASE_SCAN_HEAP, + VACUUM_ERRCB_PHASE_VACUUM_INDEX, + VACUUM_ERRCB_PHASE_VACUUM_HEAP, + VACUUM_ERRCB_PHASE_INDEX_CLEANUP, + VACUUM_ERRCB_PHASE_TRUNCATE, +} errcb_phase; + typedef struct LVRelStats { + char *relnamespace; + char *relname; /* useindex = true means two-pass strategy; false means one-pass */ bool useindex; /* Overall statistics about rel */ @@ -290,8 +302,12 @@ typedef struct LVRelStats int num_index_scans; TransactionId latestRemovedXid; bool lock_waiter_detected; -} LVRelStats; + /* Used for error callback: */ + char *indname; + BlockNumber blkno; /* used only for heap operations */ + errcb_phase phase; +} LVRelStats; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -314,10 +330,10 @@ static void lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples); + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats); static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count); + double reltuples, bool estimated_count, LVRelStats *vacrelstats); static int lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, int tupindex, LVRelStats *vacrelstats, Buffer *vmbuffer); static bool should_attempt_truncation(VacuumParams *params, @@ -337,13 +353,13 @@ static void lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult * int nindexes); static void parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVDeadTuples *dead_tuples, - int nindexes); + int nindexes, LVRelStats *vacrelstats); static void vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVSharedIndStats *shared_indstats, - LVDeadTuples *dead_tuples); + LVDeadTuples *dead_tuples, LVRelStats *vacrelstats); static void lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); @@ -361,6 +377,10 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); +static void update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, + BlockNumber blkno, char *indname, + bool free_oldindname); /* @@ -460,6 +480,10 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, vacrelstats = (LVRelStats *) palloc0(sizeof(LVRelStats)); + vacrelstats->relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats->relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats->indname = NULL; + vacrelstats->phase = VACUUM_ERRCB_PHASE_UNKNOWN; vacrelstats->old_rel_pages = onerel->rd_rel->relpages; vacrelstats->old_live_tuples = onerel->rd_rel->reltuples; vacrelstats->num_index_scans = 0; @@ -699,7 +723,6 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, BlockNumber nblocks, blkno; HeapTupleData tuple; - char *relname; TransactionId relfrozenxid = onerel->rd_rel->relfrozenxid; TransactionId relminmxid = onerel->rd_rel->relminmxid; BlockNumber empty_pages, @@ -724,20 +747,20 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); - relname = RelationGetRelationName(onerel); if (aggressive) ereport(elevel, (errmsg("aggressively vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); else ereport(elevel, (errmsg("vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); empty_pages = vacuumed_pages = 0; next_fsm_block_to_vacuum = (BlockNumber) 0; @@ -870,6 +893,12 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + errcallback.callback = vacuum_error_callback; + errcallback.arg = vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -893,6 +922,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_SCAN_HEAP, + blkno, NULL, false); + if (blkno == next_unskippable_block) { /* Time to advance next_unskippable_block */ @@ -1534,7 +1566,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, && VM_ALL_VISIBLE(onerel, blkno, &vmbuffer)) { elog(WARNING, "page is not marked all-visible but visibility map bit is set in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); visibilitymap_clear(onerel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -1555,7 +1587,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else if (PageIsAllVisible(page) && has_dead_tuples) { elog(WARNING, "page containing dead tuples is marked as all-visible in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); PageClearAllVisible(page); MarkBufferDirty(buf); visibilitymap_clear(onerel, blkno, vmbuffer, @@ -1651,6 +1683,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, if (vacrelstats->useindex) lazy_cleanup_all_indexes(Irel, indstats, vacrelstats, lps, nindexes); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* * End parallel mode before updating index statistics as we cannot write * during parallel mode. @@ -1744,7 +1779,7 @@ lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, for (idx = 0; idx < nindexes; idx++) lazy_vacuum_index(Irel[idx], &stats[idx], vacrelstats->dead_tuples, - vacrelstats->old_live_tuples); + vacrelstats->old_live_tuples, vacrelstats); } /* Increase and report the number of index scans */ @@ -1772,11 +1807,17 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) int npages; PGRUsage ru0; Buffer vmbuffer = InvalidBuffer; + LVRelStats olderrcbarg; /* Report that we are now vacuuming the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + /* Setup error traceback support for ereport() */ + olderrcbarg = *vacrelstats; + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_VACUUM_HEAP, + InvalidBlockNumber, NULL, false); + pg_rusage_init(&ru0); npages = 0; @@ -1791,6 +1832,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + vacrelstats->blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1822,6 +1864,13 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) RelationGetRelationName(onerel), tupindex, npages), errdetail_internal("%s", pg_rusage_show(&ru0)))); + + /* Revert back to the old phase information for error traceback */ + update_vacuum_error_cbarg(vacrelstats, + olderrcbarg.phase, + olderrcbarg.blkno, + olderrcbarg.indname, + true); } /* @@ -1844,9 +1893,15 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, int uncnt = 0; TransactionId visibility_cutoff_xid; bool all_frozen; + LVRelStats olderrcbarg; pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); + /* Setup error traceback support for ereport() */ + olderrcbarg = *vacrelstats; + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_VACUUM_HEAP, + blkno, NULL, false); + START_CRIT_SECTION(); for (; tupindex < dead_tuples->num_tuples; tupindex++) @@ -1923,6 +1978,12 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, *vmbuffer, visibility_cutoff_xid, flags); } + /* Revert back to the old phase information for error traceback */ + update_vacuum_error_cbarg(vacrelstats, + olderrcbarg.phase, + olderrcbarg.blkno, + olderrcbarg.indname, + true); return tupindex; } @@ -2083,7 +2144,7 @@ lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, * indexes in the case where no workers are launched. */ parallel_vacuum_index(Irel, stats, lps->lvshared, - vacrelstats->dead_tuples, nindexes); + vacrelstats->dead_tuples, nindexes, vacrelstats); /* Wait for all vacuum workers to finish */ WaitForParallelWorkersToFinish(lps->pcxt); @@ -2106,7 +2167,7 @@ lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, static void parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVDeadTuples *dead_tuples, - int nindexes) + int nindexes, LVRelStats *vacrelstats) { /* * Increment the active worker count if we are able to launch any worker. @@ -2140,7 +2201,7 @@ parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ vacuum_one_index(Irel[idx], &(stats[idx]), lvshared, shared_indstats, - dead_tuples); + dead_tuples, vacrelstats); } /* @@ -2180,7 +2241,8 @@ vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, if (shared_indstats == NULL || skip_parallel_vacuum_index(Irel[i], lps->lvshared)) vacuum_one_index(Irel[i], &(stats[i]), lps->lvshared, - shared_indstats, vacrelstats->dead_tuples); + shared_indstats, vacrelstats->dead_tuples, + vacrelstats); } /* @@ -2200,7 +2262,7 @@ vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, static void vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVSharedIndStats *shared_indstats, - LVDeadTuples *dead_tuples) + LVDeadTuples *dead_tuples, LVRelStats *vacrelstats) { IndexBulkDeleteResult *bulkdelete_res = NULL; @@ -2220,10 +2282,10 @@ vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ if (lvshared->for_cleanup) lazy_cleanup_index(indrel, stats, lvshared->reltuples, - lvshared->estimated_count); + lvshared->estimated_count, vacrelstats); else lazy_vacuum_index(indrel, stats, dead_tuples, - lvshared->reltuples); + lvshared->reltuples, vacrelstats); /* * Copy the index bulk-deletion result returned from ambulkdelete and @@ -2298,7 +2360,8 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, for (idx = 0; idx < nindexes; idx++) lazy_cleanup_index(Irel[idx], &stats[idx], vacrelstats->new_rel_tuples, - vacrelstats->tupcount_pages < vacrelstats->rel_pages); + vacrelstats->tupcount_pages < vacrelstats->rel_pages, + vacrelstats); } } @@ -2313,11 +2376,12 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, */ static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples) + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + LVRelStats olderrcbarg; pg_rusage_init(&ru0); @@ -2329,6 +2393,14 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + olderrcbarg = *vacrelstats; + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_VACUUM_INDEX, + InvalidBlockNumber, + RelationGetRelationName(indrel), + false); + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); @@ -2343,6 +2415,13 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, RelationGetRelationName(indrel), dead_tuples->num_tuples), errdetail_internal("%s", pg_rusage_show(&ru0)))); + + /* Revert back to the old phase information for error traceback */ + update_vacuum_error_cbarg(vacrelstats, + olderrcbarg.phase, + olderrcbarg.blkno, + olderrcbarg.indname, + true); } /* @@ -2354,11 +2433,12 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count) + double reltuples, bool estimated_count, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + LVRelStats olderrcbarg; pg_rusage_init(&ru0); @@ -2371,6 +2451,14 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + olderrcbarg = *vacrelstats; + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_INDEX_CLEANUP, + InvalidBlockNumber, + RelationGetRelationName(indrel), + false); + *stats = index_vacuum_cleanup(&ivinfo, *stats); if (!(*stats)) @@ -2392,6 +2480,13 @@ lazy_cleanup_index(Relation indrel, (*stats)->tuples_removed, (*stats)->pages_deleted, (*stats)->pages_free, pg_rusage_show(&ru0)))); + + /* Revert back to the old phase information for error traceback */ + update_vacuum_error_cbarg(vacrelstats, + olderrcbarg.phase, + olderrcbarg.blkno, + olderrcbarg.indname, + true); } /* @@ -2440,6 +2535,7 @@ lazy_truncate_heap(Relation onerel, LVRelStats *vacrelstats) BlockNumber old_rel_pages = vacrelstats->rel_pages; BlockNumber new_rel_pages; int lock_retry; + LVRelStats olderrcbarg; /* Report that we are now truncating */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, @@ -2510,6 +2606,12 @@ lazy_truncate_heap(Relation onerel, LVRelStats *vacrelstats) return; } + /* Setup error traceback support for ereport() */ + olderrcbarg = *vacrelstats; + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_TRUNCATE, new_rel_pages, NULL, + false); + /* * Scan backwards from the end to verify that the end pages actually * contain no tuples. This is *necessary*, not optional, because @@ -2530,6 +2632,13 @@ lazy_truncate_heap(Relation onerel, LVRelStats *vacrelstats) */ RelationTruncate(onerel, new_rel_pages); + /* Revert back to the old phase information for error traceback */ + update_vacuum_error_cbarg(vacrelstats, + olderrcbarg.phase, + olderrcbarg.blkno, + olderrcbarg.indname, + true); + /* * We can release the exclusive lock as soon as we have truncated. * Other backends can't safely access the relation until they have @@ -3320,6 +3429,8 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) int nindexes; char *sharedquery; IndexBulkDeleteResult **stats; + LVRelStats vacrelstats; + ErrorContextCallback errcallback; lvshared = (LVShared *) shm_toc_lookup(toc, PARALLEL_VACUUM_KEY_SHARED, false); @@ -3369,10 +3480,94 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) if (lvshared->maintenance_work_mem_worker > 0) maintenance_work_mem = lvshared->maintenance_work_mem_worker; + /* Init vacrelstats for use as error callback arg by parallel worker */ + vacrelstats.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats.relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats.indname = NULL; + vacrelstats.phase = VACUUM_ERRCB_PHASE_UNKNOWN; /* Not yet processing */ + + /* Setup error traceback support for ereport() */ + errcallback.callback = vacuum_error_callback; + errcallback.arg = &vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + /* Process indexes to perform vacuum/cleanup */ - parallel_vacuum_index(indrels, stats, lvshared, dead_tuples, nindexes); + parallel_vacuum_index(indrels, stats, lvshared, dead_tuples, nindexes, + &vacrelstats); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; vac_close_indexes(nindexes, indrels, RowExclusiveLock); table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + LVRelStats *cbarg = arg; + + switch (cbarg->phase) + { + case VACUUM_ERRCB_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while vacuuming block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_VACUUM_INDEX: + errcontext("while vacuuming index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_INDEX_CLEANUP: + errcontext("while cleaning up index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_TRUNCATE: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while truncating relation \"%s.%s\" to %u blocks", + cbarg->relnamespace, cbarg->relname, cbarg->blkno); + break; + + case VACUUM_ERRCB_PHASE_UNKNOWN: + default: + return; /* do nothing; the cbarg may not be + * initialized */ + } +} + +/* + * Update vacuum error callback for current phase, block and index + * + * free_oldindex is true if the previous "indname" should be freed. It must be + * false if the caller has copied the old LVRelStats, to avoid keeping a + * pointer to a freed allocation. In which case, the caller should call again + * with free_oldindname=true to avoid a leak. + */ +static void +update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, BlockNumber blkno, + char *indname, bool free_oldindname) +{ + errcbarg->blkno = blkno; + errcbarg->phase = phase; + + /* Free index name from any previous phase */ + if (free_oldindname && errcbarg->indname) + pfree(errcbarg->indname); + + /* For index phases, save the name of the current index for the callback */ + errcbarg->indname = indname ? pstrdup(indname) : NULL; +} diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index fcfcf56f4f..1dee4e1ff2 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -582,6 +582,7 @@ EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData EquivalenceClass EquivalenceMember +errcb_phase ErrorContextCallback ErrorData EstimateDSMForeignScan_function -- 2.17.0 --3C3ZMpwu25Mtx3MN Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v29-0002-Drop-reltuples.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v25 1/4] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) Discussion: https://www.postgresql.org/message-id/[email protected] --- src/backend/access/heap/vacuumlazy.c | 202 +++++++++++++++++++++++---- 1 file changed, 177 insertions(+), 25 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 03c43efc32..9f0efa5aad 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -268,8 +268,19 @@ typedef struct LVParallelState int nindexes_parallel_condcleanup; } LVParallelState; +typedef enum { + VACUUM_ERRCB_PHASE_UNKNOWN, + VACUUM_ERRCB_PHASE_SCAN_HEAP, + VACUUM_ERRCB_PHASE_VACUUM_INDEX, + VACUUM_ERRCB_PHASE_VACUUM_HEAP, + VACUUM_ERRCB_PHASE_INDEX_CLEANUP, + VACUUM_ERRCB_PHASE_VACUUM_FSM, +} errcb_phase; + typedef struct LVRelStats { + char *relnamespace; + char *relname; /* useindex = true means two-pass strategy; false means one-pass */ bool useindex; /* Overall statistics about rel */ @@ -290,8 +301,12 @@ typedef struct LVRelStats int num_index_scans; TransactionId latestRemovedXid; bool lock_waiter_detected; -} LVRelStats; + /* Used for error callback: */ + char *indname; + BlockNumber blkno; /* used only for heap operations */ + errcb_phase phase; +} LVRelStats; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -314,10 +329,10 @@ static void lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples); + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats); static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count); + double reltuples, bool estimated_count, LVRelStats *vacrelstats); static int lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, int tupindex, LVRelStats *vacrelstats, Buffer *vmbuffer); static bool should_attempt_truncation(VacuumParams *params, @@ -337,13 +352,13 @@ static void lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult * int nindexes); static void parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVDeadTuples *dead_tuples, - int nindexes); + int nindexes, LVRelStats *vacrelstats); static void vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVSharedIndStats *shared_indstats, - LVDeadTuples *dead_tuples); + LVDeadTuples *dead_tuples, LVRelStats *vacrelstats); static void lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); @@ -361,6 +376,9 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); +static void update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, + BlockNumber blkno, Relation rel); /* @@ -460,6 +478,9 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, vacrelstats = (LVRelStats *) palloc0(sizeof(LVRelStats)); + vacrelstats->relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats->relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats->indname = NULL; vacrelstats->old_rel_pages = onerel->rd_rel->relpages; vacrelstats->old_live_tuples = onerel->rd_rel->reltuples; vacrelstats->num_index_scans = 0; @@ -699,7 +720,6 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, BlockNumber nblocks, blkno; HeapTupleData tuple; - char *relname; TransactionId relfrozenxid = onerel->rd_rel->relfrozenxid; TransactionId relminmxid = onerel->rd_rel->relminmxid; BlockNumber empty_pages, @@ -724,20 +744,20 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); - relname = RelationGetRelationName(onerel); if (aggressive) ereport(elevel, (errmsg("aggressively vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); else ereport(elevel, (errmsg("vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); empty_pages = vacuumed_pages = 0; next_fsm_block_to_vacuum = (BlockNumber) 0; @@ -870,6 +890,14 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_SCAN_HEAP, + InvalidBlockNumber, NULL); + errcallback.callback = vacuum_error_callback; + errcallback.arg = vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,6 +919,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + vacrelstats->blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -1005,12 +1035,18 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, * Vacuum the Free Space Map to make newly-freed space visible on * upper-level FSM pages. Note we have not yet processed blkno. */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_VACUUM_FSM, InvalidBlockNumber, NULL); FreeSpaceMapVacuumRange(onerel, next_fsm_block_to_vacuum, blkno); next_fsm_block_to_vacuum = blkno; /* Report that we are once again scanning the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + + /* Set the error context while continuing heap scan */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_SCAN_HEAP, blkno, NULL); } /* @@ -1488,10 +1524,18 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, */ if (blkno - next_fsm_block_to_vacuum >= VACUUM_FSM_EVERY_PAGES) { + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_VACUUM_FSM, InvalidBlockNumber, + NULL); FreeSpaceMapVacuumRange(onerel, next_fsm_block_to_vacuum, blkno); next_fsm_block_to_vacuum = blkno; } + + /* Set the error context while continuing heap scan */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_SCAN_HEAP, blkno, NULL); + } freespace = PageGetHeapFreeSpace(page); @@ -1534,7 +1578,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, && VM_ALL_VISIBLE(onerel, blkno, &vmbuffer)) { elog(WARNING, "page is not marked all-visible but visibility map bit is set in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); visibilitymap_clear(onerel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -1555,7 +1599,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else if (PageIsAllVisible(page) && has_dead_tuples) { elog(WARNING, "page containing dead tuples is marked as all-visible in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); PageClearAllVisible(page); MarkBufferDirty(buf); visibilitymap_clear(onerel, blkno, vmbuffer, @@ -1642,7 +1686,11 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, * not there were indexes. */ if (blkno > next_fsm_block_to_vacuum) + { + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_VACUUM_FSM, InvalidBlockNumber, NULL); FreeSpaceMapVacuumRange(onerel, next_fsm_block_to_vacuum, blkno); + } /* report all blocks vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); @@ -1651,6 +1699,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, if (vacrelstats->useindex) lazy_cleanup_all_indexes(Irel, indstats, vacrelstats, lps, nindexes); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* * End parallel mode before updating index statistics as we cannot write * during parallel mode. @@ -1744,7 +1795,7 @@ lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, for (idx = 0; idx < nindexes; idx++) lazy_vacuum_index(Irel[idx], &stats[idx], vacrelstats->dead_tuples, - vacrelstats->old_live_tuples); + vacrelstats->old_live_tuples, vacrelstats); } /* Increase and report the number of index scans */ @@ -1777,6 +1828,10 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_VACUUM_HEAP, + InvalidBlockNumber, NULL); + pg_rusage_init(&ru0); npages = 0; @@ -1791,6 +1846,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + vacrelstats->blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1847,6 +1903,10 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_VACUUM_HEAP, + blkno, NULL); + START_CRIT_SECTION(); for (; tupindex < dead_tuples->num_tuples; tupindex++) @@ -2083,7 +2143,7 @@ lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, * indexes in the case where no workers are launched. */ parallel_vacuum_index(Irel, stats, lps->lvshared, - vacrelstats->dead_tuples, nindexes); + vacrelstats->dead_tuples, nindexes, vacrelstats); /* Wait for all vacuum workers to finish */ WaitForParallelWorkersToFinish(lps->pcxt); @@ -2106,7 +2166,7 @@ lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, static void parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVDeadTuples *dead_tuples, - int nindexes) + int nindexes, LVRelStats *vacrelstats) { /* * Increment the active worker count if we are able to launch any worker. @@ -2140,7 +2200,7 @@ parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ vacuum_one_index(Irel[idx], &(stats[idx]), lvshared, shared_indstats, - dead_tuples); + dead_tuples, vacrelstats); } /* @@ -2180,7 +2240,8 @@ vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, if (shared_indstats == NULL || skip_parallel_vacuum_index(Irel[i], lps->lvshared)) vacuum_one_index(Irel[i], &(stats[i]), lps->lvshared, - shared_indstats, vacrelstats->dead_tuples); + shared_indstats, vacrelstats->dead_tuples, + vacrelstats); } /* @@ -2200,7 +2261,7 @@ vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, static void vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVSharedIndStats *shared_indstats, - LVDeadTuples *dead_tuples) + LVDeadTuples *dead_tuples, LVRelStats *vacrelstats) { IndexBulkDeleteResult *bulkdelete_res = NULL; @@ -2220,10 +2281,10 @@ vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ if (lvshared->for_cleanup) lazy_cleanup_index(indrel, stats, lvshared->reltuples, - lvshared->estimated_count); + lvshared->estimated_count, vacrelstats); else lazy_vacuum_index(indrel, stats, dead_tuples, - lvshared->reltuples); + lvshared->reltuples, vacrelstats); /* * Copy the index bulk-deletion result returned from ambulkdelete and @@ -2298,7 +2359,8 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, for (idx = 0; idx < nindexes; idx++) lazy_cleanup_index(Irel[idx], &stats[idx], vacrelstats->new_rel_tuples, - vacrelstats->tupcount_pages < vacrelstats->rel_pages); + vacrelstats->tupcount_pages < vacrelstats->rel_pages, + vacrelstats); } } @@ -2313,7 +2375,7 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, */ static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples) + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; @@ -2329,6 +2391,11 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, + indrel); + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); @@ -2354,7 +2421,7 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count) + double reltuples, bool estimated_count, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; @@ -2371,6 +2438,10 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_INDEX_CLEANUP, InvalidBlockNumber, indrel); + *stats = index_vacuum_cleanup(&ivinfo, *stats); if (!(*stats)) @@ -3320,6 +3391,8 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) int nindexes; char *sharedquery; IndexBulkDeleteResult **stats; + LVRelStats vacrelstats; + ErrorContextCallback errcallback; lvshared = (LVShared *) shm_toc_lookup(toc, PARALLEL_VACUUM_KEY_SHARED, false); @@ -3369,10 +3442,89 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) if (lvshared->maintenance_work_mem_worker > 0) maintenance_work_mem = lvshared->maintenance_work_mem_worker; + /* Init vacrelstats for use as error callback arg by parallel worker */ + vacrelstats.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats.relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats.indname = NULL; + vacrelstats.phase = VACUUM_ERRCB_PHASE_UNKNOWN; /* Not yet processing */ + + /* Setup error traceback support for ereport() */ + errcallback.callback = vacuum_error_callback; + errcallback.arg = &vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + /* Process indexes to perform vacuum/cleanup */ - parallel_vacuum_index(indrels, stats, lvshared, dead_tuples, nindexes); + parallel_vacuum_index(indrels, stats, lvshared, dead_tuples, nindexes, + &vacrelstats); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; vac_close_indexes(nindexes, indrels, RowExclusiveLock); table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + LVRelStats *cbarg = arg; + + switch (cbarg->phase) { + case VACUUM_ERRCB_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while vacuuming block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_VACUUM_INDEX: + errcontext("while vacuuming index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_INDEX_CLEANUP: + errcontext("while cleaning up index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_VACUUM_FSM: + errcontext("while vacuuming free space map of relation \"%s.%s\"", + cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_UNKNOWN: + default: + return; /* do nothing; the cbarg maybe isn't yet initialized */ + } +} + +/* Update vacuum error callback for current phase, block and index */ +static void +update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, BlockNumber blkno, + Relation indrel) +{ + errcbarg->blkno = blkno; + errcbarg->phase = phase; + + /* Free index name from any previous phase */ + if (errcbarg->indname) { + pfree(errcbarg->indname); + errcbarg->indname = NULL; + } + + /* For index phases, save the name of the current index for the callback */ + if (indrel) { + Assert(indrel->rd_rel->relkind == RELKIND_INDEX); + errcbarg->indname = pstrdup(RelationGetRelationName(indrel)); + } +} -- 2.17.0 --EcLUEBHJhOmOUQ0C Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v25-0002-Drop-reltuples.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v19 1/2] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) Discussion: https://www.postgresql.org/message-id/[email protected] --- src/backend/access/heap/vacuumlazy.c | 130 +++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index a23cdef..5e734ee 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -292,6 +292,14 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relnamespace; + char *relname; + char *indname; + BlockNumber blkno; /* used only for heap operations */ + int phase; /* Reusing same enums as for progress reporting */ +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -361,6 +369,9 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); +static void init_vacuum_error_callback(ErrorContextCallback *errcallback, + vacuum_error_callback_arg *errcbarg, Relation onerel, int phase); /* @@ -724,6 +735,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -870,6 +883,11 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, onerel, + PROGRESS_VACUUM_PHASE_SCAN_HEAP); + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,6 +909,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -987,6 +1007,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, vmbuffer = InvalidBuffer; } + /* Pop the error context stack while calling vacuum */ + error_context_stack = errcallback.previous; + /* Work on all the indexes, then the heap */ lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); @@ -1011,6 +1034,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, /* Report that we are once again scanning the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + + /* Set the error context while continuing heap scan */ + error_context_stack = &errcallback; } /* @@ -1597,6 +1623,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -1772,6 +1801,8 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) int npages; PGRUsage ru0; Buffer vmbuffer = InvalidBuffer; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; /* Report that we are now vacuuming the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, @@ -1780,6 +1811,13 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) pg_rusage_init(&ru0); npages = 0; + /* + * Setup error traceback support for ereport() + */ + init_vacuum_error_callback(&errcallback, &errcbarg, onerel, + PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + error_context_stack = &errcallback; + tupindex = 0; while (tupindex < vacrelstats->dead_tuples->num_tuples) { @@ -1791,6 +1829,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + errcbarg.blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1811,6 +1850,9 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) npages++; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (BufferIsValid(vmbuffer)) { ReleaseBuffer(vmbuffer); @@ -2318,6 +2360,8 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -2329,10 +2373,18 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, indrel, + PROGRESS_VACUUM_PHASE_VACUUM_INDEX); + error_context_stack = &errcallback; + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (IsParallelWorker()) msg = gettext_noop("scanned index \"%s\" to remove %d row versions by parallel vacuum worker"); else @@ -2359,6 +2411,8 @@ lazy_cleanup_index(Relation indrel, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + vacuum_error_callback_arg errcbarg; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); @@ -2371,8 +2425,16 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, indrel, + PROGRESS_VACUUM_PHASE_INDEX_CLEANUP); + error_context_stack = &errcallback; + *stats = index_vacuum_cleanup(&ivinfo, *stats); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (!(*stats)) return; @@ -3375,3 +3437,71 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + + switch (cbarg->phase) { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext(_("while scanning block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext(_("while vacuuming block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + errcontext(_("while vacuuming index \"%s.%s\" of relation \"%s\""), + cbarg->relnamespace, cbarg->indname, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + errcontext(_("while cleaning up index \"%s.%s\" of relation \"%s\""), + cbarg->relnamespace, cbarg->indname, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_TRUNCATE: + case PROGRESS_VACUUM_PHASE_FINAL_CLEANUP: + default: + return; /* Shouldn't happen: do nothing */ + } +} + +/* Initialize vacuum error callback */ +static void +init_vacuum_error_callback(ErrorContextCallback *errcallback, + vacuum_error_callback_arg *errcbarg, Relation rel, int phase) +{ + switch (phase) + { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + errcbarg->relname = RelationGetRelationName(rel); + errcbarg->indname = NULL; /* Not used for heap */ + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + /* rel is an index relation in index vacuum case */ + errcbarg->relname = get_rel_name(rel->rd_index->indrelid); + errcbarg->indname = RelationGetRelationName(rel); + break; + } + + errcbarg->relnamespace = get_namespace_name(RelationGetNamespace(rel)); + errcbarg->blkno = InvalidBlockNumber; /* Not known yet */ + errcbarg->phase = phase; + + errcallback->callback = vacuum_error_callback; + errcallback->arg = errcbarg; + errcallback->previous = error_context_stack; +} -- 2.7.4 --2FkSFaIQeDFoAt0B Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v19-0002-add-callback-for-truncation.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v22 1/3] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) Discussion: https://www.postgresql.org/message-id/[email protected] --- src/backend/access/heap/vacuumlazy.c | 162 +++++++++++++++++++++++++++++++---- 1 file changed, 144 insertions(+), 18 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 03c43ef..d34d0c5 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -270,6 +270,8 @@ typedef struct LVParallelState typedef struct LVRelStats { + char *relnamespace; + char *relname; /* useindex = true means two-pass strategy; false means one-pass */ bool useindex; /* Overall statistics about rel */ @@ -290,8 +292,12 @@ typedef struct LVRelStats int num_index_scans; TransactionId latestRemovedXid; bool lock_waiter_detected; -} LVRelStats; + /* Used for error callback: */ + char *indname; + BlockNumber blkno; /* used only for heap operations */ + int phase; /* Reusing same constants as for progress reporting */ +} LVRelStats; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -314,10 +320,10 @@ static void lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples); + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats); static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count); + double reltuples, bool estimated_count, LVRelStats *vacrelstats); static int lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, int tupindex, LVRelStats *vacrelstats, Buffer *vmbuffer); static bool should_attempt_truncation(VacuumParams *params, @@ -361,6 +367,9 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); +static void update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, + BlockNumber blkno, Relation rel); /* @@ -460,6 +469,9 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, vacrelstats = (LVRelStats *) palloc0(sizeof(LVRelStats)); + vacrelstats->relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats->relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats->indname = NULL; vacrelstats->old_rel_pages = onerel->rd_rel->relpages; vacrelstats->old_live_tuples = onerel->rd_rel->reltuples; vacrelstats->num_index_scans = 0; @@ -699,7 +711,6 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, BlockNumber nblocks, blkno; HeapTupleData tuple; - char *relname; TransactionId relfrozenxid = onerel->rd_rel->relfrozenxid; TransactionId relminmxid = onerel->rd_rel->relminmxid; BlockNumber empty_pages, @@ -724,20 +735,20 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); - relname = RelationGetRelationName(onerel); if (aggressive) ereport(elevel, (errmsg("aggressively vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); else ereport(elevel, (errmsg("vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); empty_pages = vacuumed_pages = 0; next_fsm_block_to_vacuum = (BlockNumber) 0; @@ -870,6 +881,14 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, PROGRESS_VACUUM_PHASE_SCAN_HEAP, + InvalidBlockNumber, NULL); + errcallback.callback = vacuum_error_callback; + errcallback.arg = vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,6 +910,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + vacrelstats->blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -1011,6 +1032,10 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, /* Report that we are once again scanning the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + + /* Set the error context while continuing heap scan */ + update_vacuum_error_cbarg(vacrelstats, + PROGRESS_VACUUM_PHASE_SCAN_HEAP, blkno, NULL); } /* @@ -1534,7 +1559,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, && VM_ALL_VISIBLE(onerel, blkno, &vmbuffer)) { elog(WARNING, "page is not marked all-visible but visibility map bit is set in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); visibilitymap_clear(onerel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -1555,7 +1580,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else if (PageIsAllVisible(page) && has_dead_tuples) { elog(WARNING, "page containing dead tuples is marked as all-visible in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); PageClearAllVisible(page); MarkBufferDirty(buf); visibilitymap_clear(onerel, blkno, vmbuffer, @@ -1651,6 +1676,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, if (vacrelstats->useindex) lazy_cleanup_all_indexes(Irel, indstats, vacrelstats, lps, nindexes); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* * End parallel mode before updating index statistics as we cannot write * during parallel mode. @@ -1744,7 +1772,7 @@ lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, for (idx = 0; idx < nindexes; idx++) lazy_vacuum_index(Irel[idx], &stats[idx], vacrelstats->dead_tuples, - vacrelstats->old_live_tuples); + vacrelstats->old_live_tuples, vacrelstats); } /* Increase and report the number of index scans */ @@ -1778,8 +1806,13 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) PROGRESS_VACUUM_PHASE_VACUUM_HEAP); pg_rusage_init(&ru0); - npages = 0; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + PROGRESS_VACUUM_PHASE_VACUUM_HEAP, InvalidBlockNumber, + NULL); + + npages = 0; tupindex = 0; while (tupindex < vacrelstats->dead_tuples->num_tuples) { @@ -1791,6 +1824,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + vacrelstats->blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -2203,6 +2237,26 @@ vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVDeadTuples *dead_tuples) { IndexBulkDeleteResult *bulkdelete_res = NULL; + /* Init vacrelstats for error callback by parallel workers: */ + LVRelStats vacrelstats = + { + .relnamespace = get_namespace_name(RelationGetNamespace(indrel)), + .relname = pstrdup(get_rel_name(indrel->rd_index->indrelid)), + .indname = pstrdup(RelationGetRelationName(indrel)), + }; + + /* Setup error traceback support for ereport() */ + ErrorContextCallback errcallback; + update_vacuum_error_cbarg(&vacrelstats, + lvshared->for_cleanup ? + PROGRESS_VACUUM_PHASE_INDEX_CLEANUP : + PROGRESS_VACUUM_PHASE_VACUUM_INDEX, + InvalidBlockNumber, NULL); + + errcallback.callback = vacuum_error_callback; + errcallback.arg = &vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; if (shared_indstats) { @@ -2220,10 +2274,13 @@ vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ if (lvshared->for_cleanup) lazy_cleanup_index(indrel, stats, lvshared->reltuples, - lvshared->estimated_count); + lvshared->estimated_count, &vacrelstats); else lazy_vacuum_index(indrel, stats, dead_tuples, - lvshared->reltuples); + lvshared->reltuples, &vacrelstats); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; /* * Copy the index bulk-deletion result returned from ambulkdelete and @@ -2298,7 +2355,8 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, for (idx = 0; idx < nindexes; idx++) lazy_cleanup_index(Irel[idx], &stats[idx], vacrelstats->new_rel_tuples, - vacrelstats->tupcount_pages < vacrelstats->rel_pages); + vacrelstats->tupcount_pages < vacrelstats->rel_pages, + vacrelstats); } } @@ -2313,7 +2371,7 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, */ static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples) + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; @@ -2329,6 +2387,11 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + PROGRESS_VACUUM_PHASE_VACUUM_INDEX, InvalidBlockNumber, + indrel); + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); @@ -2354,7 +2417,7 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count) + double reltuples, bool estimated_count, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; @@ -2371,6 +2434,10 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + PROGRESS_VACUUM_PHASE_INDEX_CLEANUP, InvalidBlockNumber, indrel); + *stats = index_vacuum_cleanup(&ivinfo, *stats); if (!(*stats)) @@ -3376,3 +3443,62 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + LVRelStats *cbarg = arg; + + switch (cbarg->phase) { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while vacuuming block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + errcontext("while vacuuming index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + errcontext("while cleaning up index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_TRUNCATE: + case PROGRESS_VACUUM_PHASE_FINAL_CLEANUP: + default: + return; /* Shouldn't happen: do nothing */ + } +} + +/* Update vacuum error callback for current phase, block and index */ +static void +update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, BlockNumber blkno, + Relation indrel) +{ + errcbarg->blkno = blkno; + errcbarg->phase = phase; + + /* Free index name from any previous phase */ + if (errcbarg->indname) { + pfree(errcbarg->indname); + errcbarg->indname = NULL; + } + + /* For index phases, save the name of the current index for the callback */ + if (indrel) { + Assert(indrel->rd_rel->relkind == RELKIND_INDEX); + errcbarg->indname = pstrdup(RelationGetRelationName(indrel)); + } +} -- 2.7.4 --Q68bSM7Ycu6FN28Q Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v22-0002-add-callback-for-truncation.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v23 1/3] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) Discussion: https://www.postgresql.org/message-id/[email protected] --- src/backend/access/heap/vacuumlazy.c | 178 +++++++++++++++++++++++---- src/include/commands/progress.h | 1 + 2 files changed, 153 insertions(+), 26 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 03c43efc32..f15326a24c 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -270,6 +270,8 @@ typedef struct LVParallelState typedef struct LVRelStats { + char *relnamespace; + char *relname; /* useindex = true means two-pass strategy; false means one-pass */ bool useindex; /* Overall statistics about rel */ @@ -290,8 +292,12 @@ typedef struct LVRelStats int num_index_scans; TransactionId latestRemovedXid; bool lock_waiter_detected; -} LVRelStats; + /* Used for error callback: */ + char *indname; + BlockNumber blkno; /* used only for heap operations */ + int phase; /* Reusing same constants as for progress reporting */ +} LVRelStats; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -314,10 +320,10 @@ static void lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples); + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats); static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count); + double reltuples, bool estimated_count, LVRelStats *vacrelstats); static int lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, int tupindex, LVRelStats *vacrelstats, Buffer *vmbuffer); static bool should_attempt_truncation(VacuumParams *params, @@ -337,13 +343,13 @@ static void lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult * int nindexes); static void parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVDeadTuples *dead_tuples, - int nindexes); + int nindexes, LVRelStats *vacrelstats); static void vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVSharedIndStats *shared_indstats, - LVDeadTuples *dead_tuples); + LVDeadTuples *dead_tuples, LVRelStats *vacrelstats); static void lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); @@ -361,6 +367,9 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); +static void update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, + BlockNumber blkno, Relation rel); /* @@ -460,6 +469,9 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, vacrelstats = (LVRelStats *) palloc0(sizeof(LVRelStats)); + vacrelstats->relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats->relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats->indname = NULL; vacrelstats->old_rel_pages = onerel->rd_rel->relpages; vacrelstats->old_live_tuples = onerel->rd_rel->reltuples; vacrelstats->num_index_scans = 0; @@ -699,7 +711,6 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, BlockNumber nblocks, blkno; HeapTupleData tuple; - char *relname; TransactionId relfrozenxid = onerel->rd_rel->relfrozenxid; TransactionId relminmxid = onerel->rd_rel->relminmxid; BlockNumber empty_pages, @@ -724,20 +735,20 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); - relname = RelationGetRelationName(onerel); if (aggressive) ereport(elevel, (errmsg("aggressively vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); else ereport(elevel, (errmsg("vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); empty_pages = vacuumed_pages = 0; next_fsm_block_to_vacuum = (BlockNumber) 0; @@ -870,6 +881,14 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, PROGRESS_VACUUM_PHASE_SCAN_HEAP, + InvalidBlockNumber, NULL); + errcallback.callback = vacuum_error_callback; + errcallback.arg = vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,6 +910,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + vacrelstats->blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -1005,12 +1026,18 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, * Vacuum the Free Space Map to make newly-freed space visible on * upper-level FSM pages. Note we have not yet processed blkno. */ + update_vacuum_error_cbarg(vacrelstats, + PROGRESS_VACUUM_PHASE_VACUUM_FSM, InvalidBlockNumber, NULL); FreeSpaceMapVacuumRange(onerel, next_fsm_block_to_vacuum, blkno); next_fsm_block_to_vacuum = blkno; /* Report that we are once again scanning the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + + /* Set the error context while continuing heap scan */ + update_vacuum_error_cbarg(vacrelstats, + PROGRESS_VACUUM_PHASE_SCAN_HEAP, blkno, NULL); } /* @@ -1534,7 +1561,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, && VM_ALL_VISIBLE(onerel, blkno, &vmbuffer)) { elog(WARNING, "page is not marked all-visible but visibility map bit is set in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); visibilitymap_clear(onerel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -1555,7 +1582,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else if (PageIsAllVisible(page) && has_dead_tuples) { elog(WARNING, "page containing dead tuples is marked as all-visible in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); PageClearAllVisible(page); MarkBufferDirty(buf); visibilitymap_clear(onerel, blkno, vmbuffer, @@ -1651,6 +1678,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, if (vacrelstats->useindex) lazy_cleanup_all_indexes(Irel, indstats, vacrelstats, lps, nindexes); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* * End parallel mode before updating index statistics as we cannot write * during parallel mode. @@ -1744,7 +1774,7 @@ lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, for (idx = 0; idx < nindexes; idx++) lazy_vacuum_index(Irel[idx], &stats[idx], vacrelstats->dead_tuples, - vacrelstats->old_live_tuples); + vacrelstats->old_live_tuples, vacrelstats); } /* Increase and report the number of index scans */ @@ -1778,8 +1808,13 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) PROGRESS_VACUUM_PHASE_VACUUM_HEAP); pg_rusage_init(&ru0); - npages = 0; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + PROGRESS_VACUUM_PHASE_VACUUM_HEAP, InvalidBlockNumber, + NULL); + + npages = 0; tupindex = 0; while (tupindex < vacrelstats->dead_tuples->num_tuples) { @@ -1791,6 +1826,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + vacrelstats->blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -2083,7 +2119,7 @@ lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, * indexes in the case where no workers are launched. */ parallel_vacuum_index(Irel, stats, lps->lvshared, - vacrelstats->dead_tuples, nindexes); + vacrelstats->dead_tuples, nindexes, vacrelstats); /* Wait for all vacuum workers to finish */ WaitForParallelWorkersToFinish(lps->pcxt); @@ -2106,7 +2142,7 @@ lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, static void parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVDeadTuples *dead_tuples, - int nindexes) + int nindexes, LVRelStats *vacrelstats) { /* * Increment the active worker count if we are able to launch any worker. @@ -2140,7 +2176,7 @@ parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ vacuum_one_index(Irel[idx], &(stats[idx]), lvshared, shared_indstats, - dead_tuples); + dead_tuples, vacrelstats); } /* @@ -2180,7 +2216,8 @@ vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, if (shared_indstats == NULL || skip_parallel_vacuum_index(Irel[i], lps->lvshared)) vacuum_one_index(Irel[i], &(stats[i]), lps->lvshared, - shared_indstats, vacrelstats->dead_tuples); + shared_indstats, vacrelstats->dead_tuples, + vacrelstats); } /* @@ -2200,7 +2237,7 @@ vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, static void vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVSharedIndStats *shared_indstats, - LVDeadTuples *dead_tuples) + LVDeadTuples *dead_tuples, LVRelStats *vacrelstats) { IndexBulkDeleteResult *bulkdelete_res = NULL; @@ -2220,10 +2257,10 @@ vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ if (lvshared->for_cleanup) lazy_cleanup_index(indrel, stats, lvshared->reltuples, - lvshared->estimated_count); + lvshared->estimated_count, vacrelstats); else lazy_vacuum_index(indrel, stats, dead_tuples, - lvshared->reltuples); + lvshared->reltuples, vacrelstats); /* * Copy the index bulk-deletion result returned from ambulkdelete and @@ -2298,7 +2335,8 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, for (idx = 0; idx < nindexes; idx++) lazy_cleanup_index(Irel[idx], &stats[idx], vacrelstats->new_rel_tuples, - vacrelstats->tupcount_pages < vacrelstats->rel_pages); + vacrelstats->tupcount_pages < vacrelstats->rel_pages, + vacrelstats); } } @@ -2313,7 +2351,7 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, */ static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples) + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; @@ -2329,6 +2367,11 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + PROGRESS_VACUUM_PHASE_VACUUM_INDEX, InvalidBlockNumber, + indrel); + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); @@ -2354,7 +2397,7 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count) + double reltuples, bool estimated_count, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; @@ -2371,6 +2414,10 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + PROGRESS_VACUUM_PHASE_INDEX_CLEANUP, InvalidBlockNumber, indrel); + *stats = index_vacuum_cleanup(&ivinfo, *stats); if (!(*stats)) @@ -3320,6 +3367,8 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) int nindexes; char *sharedquery; IndexBulkDeleteResult **stats; + LVRelStats vacrelstats; + ErrorContextCallback errcallback; lvshared = (LVShared *) shm_toc_lookup(toc, PARALLEL_VACUUM_KEY_SHARED, false); @@ -3341,6 +3390,18 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) */ onerel = table_open(lvshared->relid, ShareUpdateExclusiveLock); + /* Init vacrelstats for use as error callback by parallel worker: */ + vacrelstats.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats.relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats.indname = NULL; + vacrelstats.phase = -1, /* Not yet processing */ + + /* Setup error traceback support for ereport() */ + errcallback.callback = vacuum_error_callback; + errcallback.arg = &vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + /* * Open all indexes. indrels are sorted in order by OID, which should be * matched to the leader's one. @@ -3370,9 +3431,74 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) maintenance_work_mem = lvshared->maintenance_work_mem_worker; /* Process indexes to perform vacuum/cleanup */ - parallel_vacuum_index(indrels, stats, lvshared, dead_tuples, nindexes); + parallel_vacuum_index(indrels, stats, lvshared, dead_tuples, nindexes, &vacrelstats); vac_close_indexes(nindexes, indrels, RowExclusiveLock); table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + LVRelStats *cbarg = arg; + + switch (cbarg->phase) { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while vacuuming block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + errcontext("while vacuuming index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + errcontext("while cleaning up index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_FSM: + errcontext("while vacuuming free space map of relation \"%s.%s\"", + cbarg->relnamespace, cbarg->relname); + break; + + case -1: /* Parallel worker not yet processing */ + case PROGRESS_VACUUM_PHASE_TRUNCATE: + case PROGRESS_VACUUM_PHASE_FINAL_CLEANUP: + default: + return; /* Shouldn't happen: do nothing */ + } +} + +/* Update vacuum error callback for current phase, block and index */ +static void +update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, BlockNumber blkno, + Relation indrel) +{ + errcbarg->blkno = blkno; + errcbarg->phase = phase; + + /* Free index name from any previous phase */ + if (errcbarg->indname) { + pfree(errcbarg->indname); + errcbarg->indname = NULL; + } + + /* For index phases, save the name of the current index for the callback */ + if (indrel) { + Assert(indrel->rd_rel->relkind == RELKIND_INDEX); + errcbarg->indname = pstrdup(RelationGetRelationName(indrel)); + } +} diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h index 12e9d3d42f..5af2b8cc0c 100644 --- a/src/include/commands/progress.h +++ b/src/include/commands/progress.h @@ -33,6 +33,7 @@ #define PROGRESS_VACUUM_PHASE_INDEX_CLEANUP 4 #define PROGRESS_VACUUM_PHASE_TRUNCATE 5 #define PROGRESS_VACUUM_PHASE_FINAL_CLEANUP 6 +#define PROGRESS_VACUUM_PHASE_VACUUM_FSM 7 /* For error reporting only */ /* Progress parameters for analyze */ #define PROGRESS_ANALYZE_PHASE 0 -- 2.17.0 --HcAYCG3uE/tztfnV Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v23-0002-add-callback-for-truncation.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v15 1/2] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 85 +++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 8ce5011..d11c7af 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -292,6 +292,13 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relnamespace; + char *relname; + BlockNumber blkno; + int stage; /* 0: scan heap; 1: vacuum heap; 2: vacuum index */ +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -361,6 +368,7 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); /* @@ -724,6 +732,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -870,6 +880,17 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + errcbarg.relname = relname; + errcbarg.blkno = InvalidBlockNumber; /* Not known yet */ + errcbarg.stage = 0; + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,6 +912,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -987,13 +1010,18 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, vmbuffer = InvalidBuffer; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* Work on all the indexes, then the heap */ lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); - /* Remove tuples from heap */ lazy_vacuum_heap(onerel, vacrelstats); + /* Replace error context while continuing heap scan */ + error_context_stack = &errcallback; + /* * Forget the now-vacuumed tuples, and press on, but be careful * not to reset latestRemovedXid since we want that value to be @@ -1597,6 +1625,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -1772,11 +1803,25 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) int npages; PGRUsage ru0; Buffer vmbuffer = InvalidBuffer; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; /* Report that we are now vacuuming the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + /* + * Setup error traceback support for ereport() + * ->relnamespace and ->relname are already set + */ + errcbarg.blkno = InvalidBlockNumber; /* Not known yet */ + errcbarg.stage = 1; + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + pg_rusage_init(&ru0); npages = 0; @@ -1791,6 +1836,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + errcbarg.blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1811,6 +1857,9 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) npages++; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (BufferIsValid(vmbuffer)) { ReleaseBuffer(vmbuffer); @@ -2318,6 +2367,8 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -2329,10 +2380,23 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(indrel)); + errcbarg.relname = RelationGetRelationName(indrel); + errcbarg.stage = 2; + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (IsParallelWorker()) msg = gettext_noop("scanned index \"%s\" to remove %d row versions by parallel vacuum worker"); else @@ -3375,3 +3439,22 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + + if (cbarg->stage == 0) + errcontext(_("while scanning block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + else if (cbarg->stage == 1) + errcontext(_("while vacuuming block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + else if (cbarg->stage == 2) + errcontext(_("while vacuuming index \"%s.%s\""), + cbarg->relnamespace, cbarg->relname); +} -- 2.7.4 --OFtM20PmL5gDcvsL Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v15-0002-Include-name-of-table-in-callback-for-index-vacu.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v8 4/5] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index e35fe6a..52602e2 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -138,6 +138,12 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relname; + char *relnamespace; + BlockNumber blkno; +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -178,6 +184,7 @@ static bool lazy_tid_reaped(ItemPointer itemptr, void *state); static int vac_cmp_itemptr(const void *left, const void *right); static bool heap_page_is_all_visible(Relation rel, Buffer buf, TransactionId *visibility_cutoff_xid, bool *all_frozen); +static void vacuum_error_callback(void *arg); /* @@ -609,6 +616,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -650,6 +659,15 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, next_unskippable_block = 0; skipping_blocks = skip_blocks(onerel, params, &next_unskippable_block, nblocks, &vmbuffer, aggressive); + /* Setup error traceback support for ereport() */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + errcbarg.relname = relname; + errcbarg.blkno = InvalidBlockNumber; /* Not known yet */ + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -671,6 +689,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -737,8 +757,11 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, } /* Work on all the indexes, then the heap */ + /* Don't use the errcontext handler outside this function */ + error_context_stack = errcallback.previous; lazy_vacuum_all_indexes(onerel, vacrelstats, Irel, nindexes, indstats); + error_context_stack = &errcallback; /* Remove tuples from heap */ lazy_vacuum_heap(onerel, vacrelstats); @@ -1346,6 +1369,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -2323,3 +2349,14 @@ heap_page_is_all_visible(Relation rel, Buffer buf, return all_visible; } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); +} -- 2.7.4 --tThc/1wpZn/ma/RB Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v8-0005-add-errcontext-callback-in-lazy_vacuum_heap-too.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v10 1/5] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 38 ++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index bd2e7fb..2eb3caa 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -290,8 +290,12 @@ typedef struct LVRelStats int num_index_scans; TransactionId latestRemovedXid; bool lock_waiter_detected; -} LVRelStats; + /* Used by the error callback */ + char *relname; + char *relnamespace; + BlockNumber blkno; +} LVRelStats; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -361,6 +365,7 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); /* @@ -813,6 +818,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); @@ -892,6 +898,16 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, next_unskippable_block = 0; skipping_blocks = skip_blocks(onerel, params, &next_unskippable_block, nblocks, &vmbuffer, aggressive); + /* Setup error traceback support for ereport() */ + vacrelstats->relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats->relname = relname; + vacrelstats->blkno = InvalidBlockNumber; /* Not known yet */ + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -913,6 +929,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + vacrelstats->blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -979,11 +997,13 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, } /* Work on all the indexes, then the heap */ + /* Don't use the errcontext handler outside this function */ + error_context_stack = errcallback.previous; lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); - /* Remove tuples from heap */ lazy_vacuum_heap(onerel, vacrelstats); + error_context_stack = &errcallback; /* * Forget the now-vacuumed tuples, and press on, but be careful @@ -1602,6 +1622,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -3416,3 +3439,14 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + LVRelStats *cbarg = arg; + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); +} -- 2.7.4 --qZTZQd/MRfnLEAAX Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v10-0002-add-errcontext-callback-in-lazy_vacuum_heap-too.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v12 1/5] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 38 ++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index b331f4c..0c4ec7b 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -287,8 +287,12 @@ typedef struct LVRelStats int num_index_scans; TransactionId latestRemovedXid; bool lock_waiter_detected; -} LVRelStats; + /* Used by the error callback */ + char *relname; + char *relnamespace; + BlockNumber blkno; +} LVRelStats; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -358,6 +362,7 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); /* @@ -721,6 +726,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); @@ -867,6 +873,16 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + vacrelstats->relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats->relname = relname; + vacrelstats->blkno = InvalidBlockNumber; /* Not known yet */ + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -888,6 +904,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + vacrelstats->blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -985,11 +1003,13 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, } /* Work on all the indexes, then the heap */ + /* Don't use the errcontext handler outside this function */ + error_context_stack = errcallback.previous; lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); - /* Remove tuples from heap */ lazy_vacuum_heap(onerel, vacrelstats); + error_context_stack = &errcallback; /* * Forget the now-vacuumed tuples, and press on, but be careful @@ -1594,6 +1614,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -3372,3 +3395,14 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + LVRelStats *cbarg = arg; + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); +} -- 2.7.4 --Y46NoIcKQuicSz3X Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0002-add-errcontext-callback-in-lazy_vacuum_heap-too.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v13 1/5] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 41 ++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index b331f4c..822fa3d 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -287,8 +287,12 @@ typedef struct LVRelStats int num_index_scans; TransactionId latestRemovedXid; bool lock_waiter_detected; -} LVRelStats; + /* Used by the error callback */ + char *relname; + char *relnamespace; + BlockNumber blkno; +} LVRelStats; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -358,6 +362,7 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); /* @@ -721,6 +726,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); @@ -867,6 +873,16 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + vacrelstats->relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats->relname = relname; + vacrelstats->blkno = InvalidBlockNumber; /* Not known yet */ + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -888,6 +904,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + vacrelstats->blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -984,13 +1002,18 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, vmbuffer = InvalidBuffer; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* Work on all the indexes, then the heap */ lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); - /* Remove tuples from heap */ lazy_vacuum_heap(onerel, vacrelstats); + /* Replace error context while continuing heap scan */ + error_context_stack = &errcallback; + /* * Forget the now-vacuumed tuples, and press on, but be careful * not to reset latestRemovedXid since we want that value to be @@ -1594,6 +1617,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -3372,3 +3398,14 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + LVRelStats *cbarg = arg; + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); +} -- 2.7.4 --3yNHWXBV/QO9xKNm Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-add-errcontext-callback-in-lazy_vacuum_heap-too.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v27 1/5] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) Discussion: https://www.postgresql.org/message-id/[email protected] --- src/backend/access/heap/vacuumlazy.c | 215 +++++++++++++++++++++++---- src/tools/pgindent/typedefs.list | 1 + 2 files changed, 191 insertions(+), 25 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 03c43efc32..92bac9a24d 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -268,8 +268,19 @@ typedef struct LVParallelState int nindexes_parallel_condcleanup; } LVParallelState; +typedef enum +{ + VACUUM_ERRCB_PHASE_UNKNOWN, + VACUUM_ERRCB_PHASE_SCAN_HEAP, + VACUUM_ERRCB_PHASE_VACUUM_INDEX, + VACUUM_ERRCB_PHASE_VACUUM_HEAP, + VACUUM_ERRCB_PHASE_INDEX_CLEANUP, +} errcb_phase; + typedef struct LVRelStats { + char *relnamespace; + char *relname; /* useindex = true means two-pass strategy; false means one-pass */ bool useindex; /* Overall statistics about rel */ @@ -290,8 +301,12 @@ typedef struct LVRelStats int num_index_scans; TransactionId latestRemovedXid; bool lock_waiter_detected; -} LVRelStats; + /* Used for error callback: */ + char *indname; + BlockNumber blkno; /* used only for heap operations */ + errcb_phase phase; +} LVRelStats; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -314,10 +329,10 @@ static void lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples); + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats); static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count); + double reltuples, bool estimated_count, LVRelStats *vacrelstats); static int lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, int tupindex, LVRelStats *vacrelstats, Buffer *vmbuffer); static bool should_attempt_truncation(VacuumParams *params, @@ -337,13 +352,13 @@ static void lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult * int nindexes); static void parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVDeadTuples *dead_tuples, - int nindexes); + int nindexes, LVRelStats *vacrelstats); static void vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVSharedIndStats *shared_indstats, - LVDeadTuples *dead_tuples); + LVDeadTuples *dead_tuples, LVRelStats *vacrelstats); static void lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); @@ -361,6 +376,9 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); +static void update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, + BlockNumber blkno, Relation rel); /* @@ -460,6 +478,9 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, vacrelstats = (LVRelStats *) palloc0(sizeof(LVRelStats)); + vacrelstats->relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats->relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats->indname = NULL; vacrelstats->old_rel_pages = onerel->rd_rel->relpages; vacrelstats->old_live_tuples = onerel->rd_rel->reltuples; vacrelstats->num_index_scans = 0; @@ -699,7 +720,6 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, BlockNumber nblocks, blkno; HeapTupleData tuple; - char *relname; TransactionId relfrozenxid = onerel->rd_rel->relfrozenxid; TransactionId relminmxid = onerel->rd_rel->relminmxid; BlockNumber empty_pages, @@ -724,20 +744,20 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); - relname = RelationGetRelationName(onerel); if (aggressive) ereport(elevel, (errmsg("aggressively vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); else ereport(elevel, (errmsg("vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); empty_pages = vacuumed_pages = 0; next_fsm_block_to_vacuum = (BlockNumber) 0; @@ -870,6 +890,14 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_SCAN_HEAP, + InvalidBlockNumber, NULL); + errcallback.callback = vacuum_error_callback; + errcallback.arg = vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,8 +919,13 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + vacrelstats->blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_SCAN_HEAP, + blkno, NULL); + if (blkno == next_unskippable_block) { /* Time to advance next_unskippable_block */ @@ -1011,6 +1044,10 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, /* Report that we are once again scanning the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + + /* Set the error context while continuing heap scan */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_SCAN_HEAP, blkno, NULL); } /* @@ -1492,6 +1529,11 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, blkno); next_fsm_block_to_vacuum = blkno; } + + /* Set the error context while continuing heap scan */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_SCAN_HEAP, blkno, NULL); + } freespace = PageGetHeapFreeSpace(page); @@ -1534,7 +1576,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, && VM_ALL_VISIBLE(onerel, blkno, &vmbuffer)) { elog(WARNING, "page is not marked all-visible but visibility map bit is set in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); visibilitymap_clear(onerel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -1555,7 +1597,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else if (PageIsAllVisible(page) && has_dead_tuples) { elog(WARNING, "page containing dead tuples is marked as all-visible in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); PageClearAllVisible(page); MarkBufferDirty(buf); visibilitymap_clear(onerel, blkno, vmbuffer, @@ -1651,6 +1693,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, if (vacrelstats->useindex) lazy_cleanup_all_indexes(Irel, indstats, vacrelstats, lps, nindexes); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* * End parallel mode before updating index statistics as we cannot write * during parallel mode. @@ -1744,7 +1789,7 @@ lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, for (idx = 0; idx < nindexes; idx++) lazy_vacuum_index(Irel[idx], &stats[idx], vacrelstats->dead_tuples, - vacrelstats->old_live_tuples); + vacrelstats->old_live_tuples, vacrelstats); } /* Increase and report the number of index scans */ @@ -1777,6 +1822,10 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_VACUUM_HEAP, + InvalidBlockNumber, NULL); + pg_rusage_init(&ru0); npages = 0; @@ -1791,6 +1840,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + vacrelstats->blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1822,6 +1872,11 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) RelationGetRelationName(onerel), tupindex, npages), errdetail_internal("%s", pg_rusage_show(&ru0)))); + + /* Clear the error traceback phase */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_UNKNOWN, InvalidBlockNumber, + NULL); } /* @@ -1847,6 +1902,10 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_VACUUM_HEAP, + blkno, NULL); + START_CRIT_SECTION(); for (; tupindex < dead_tuples->num_tuples; tupindex++) @@ -1923,6 +1982,11 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, *vmbuffer, visibility_cutoff_xid, flags); } + /* Clear the error traceback phase */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_UNKNOWN, InvalidBlockNumber, + NULL); + return tupindex; } @@ -2083,7 +2147,7 @@ lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, * indexes in the case where no workers are launched. */ parallel_vacuum_index(Irel, stats, lps->lvshared, - vacrelstats->dead_tuples, nindexes); + vacrelstats->dead_tuples, nindexes, vacrelstats); /* Wait for all vacuum workers to finish */ WaitForParallelWorkersToFinish(lps->pcxt); @@ -2106,7 +2170,7 @@ lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, static void parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVDeadTuples *dead_tuples, - int nindexes) + int nindexes, LVRelStats *vacrelstats) { /* * Increment the active worker count if we are able to launch any worker. @@ -2140,7 +2204,7 @@ parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ vacuum_one_index(Irel[idx], &(stats[idx]), lvshared, shared_indstats, - dead_tuples); + dead_tuples, vacrelstats); } /* @@ -2180,7 +2244,8 @@ vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, if (shared_indstats == NULL || skip_parallel_vacuum_index(Irel[i], lps->lvshared)) vacuum_one_index(Irel[i], &(stats[i]), lps->lvshared, - shared_indstats, vacrelstats->dead_tuples); + shared_indstats, vacrelstats->dead_tuples, + vacrelstats); } /* @@ -2200,7 +2265,7 @@ vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, static void vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVSharedIndStats *shared_indstats, - LVDeadTuples *dead_tuples) + LVDeadTuples *dead_tuples, LVRelStats *vacrelstats) { IndexBulkDeleteResult *bulkdelete_res = NULL; @@ -2220,10 +2285,10 @@ vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ if (lvshared->for_cleanup) lazy_cleanup_index(indrel, stats, lvshared->reltuples, - lvshared->estimated_count); + lvshared->estimated_count, vacrelstats); else lazy_vacuum_index(indrel, stats, dead_tuples, - lvshared->reltuples); + lvshared->reltuples, vacrelstats); /* * Copy the index bulk-deletion result returned from ambulkdelete and @@ -2298,7 +2363,8 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, for (idx = 0; idx < nindexes; idx++) lazy_cleanup_index(Irel[idx], &stats[idx], vacrelstats->new_rel_tuples, - vacrelstats->tupcount_pages < vacrelstats->rel_pages); + vacrelstats->tupcount_pages < vacrelstats->rel_pages, + vacrelstats); } } @@ -2313,7 +2379,7 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, */ static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples) + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; @@ -2329,6 +2395,11 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, + indrel); + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); @@ -2343,6 +2414,11 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, RelationGetRelationName(indrel), dead_tuples->num_tuples), errdetail_internal("%s", pg_rusage_show(&ru0)))); + + /* Clear the error traceback phase */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_UNKNOWN, InvalidBlockNumber, + NULL); } /* @@ -2354,7 +2430,7 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count) + double reltuples, bool estimated_count, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; @@ -2371,6 +2447,10 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_INDEX_CLEANUP, InvalidBlockNumber, indrel); + *stats = index_vacuum_cleanup(&ivinfo, *stats); if (!(*stats)) @@ -2392,6 +2472,11 @@ lazy_cleanup_index(Relation indrel, (*stats)->tuples_removed, (*stats)->pages_deleted, (*stats)->pages_free, pg_rusage_show(&ru0)))); + + /* Clear the error traceback phase */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_UNKNOWN, InvalidBlockNumber, + NULL); } /* @@ -3320,6 +3405,8 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) int nindexes; char *sharedquery; IndexBulkDeleteResult **stats; + LVRelStats vacrelstats; + ErrorContextCallback errcallback; lvshared = (LVShared *) shm_toc_lookup(toc, PARALLEL_VACUUM_KEY_SHARED, false); @@ -3369,10 +3456,88 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) if (lvshared->maintenance_work_mem_worker > 0) maintenance_work_mem = lvshared->maintenance_work_mem_worker; + /* Init vacrelstats for use as error callback arg by parallel worker */ + vacrelstats.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats.relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats.indname = NULL; + vacrelstats.phase = VACUUM_ERRCB_PHASE_UNKNOWN; /* Not yet processing */ + + /* Setup error traceback support for ereport() */ + errcallback.callback = vacuum_error_callback; + errcallback.arg = &vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + /* Process indexes to perform vacuum/cleanup */ - parallel_vacuum_index(indrels, stats, lvshared, dead_tuples, nindexes); + parallel_vacuum_index(indrels, stats, lvshared, dead_tuples, nindexes, + &vacrelstats); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; vac_close_indexes(nindexes, indrels, RowExclusiveLock); table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + LVRelStats *cbarg = arg; + + switch (cbarg->phase) + { + case VACUUM_ERRCB_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while vacuuming block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_VACUUM_INDEX: + errcontext("while vacuuming index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_INDEX_CLEANUP: + errcontext("while cleaning up index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_UNKNOWN: + default: + return; /* do nothing; the cbarg may not be + * initialized */ + } +} + +/* Update vacuum error callback for current phase, block and index */ +static void +update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, BlockNumber blkno, + Relation indrel) +{ + errcbarg->blkno = blkno; + errcbarg->phase = phase; + + /* Free index name from any previous phase */ + if (errcbarg->indname) + { + pfree(errcbarg->indname); + errcbarg->indname = NULL; + } + + /* For index phases, save the name of the current index for the callback */ + if (indrel) + { + Assert(indrel->rd_rel->relkind == RELKIND_INDEX); + errcbarg->indname = pstrdup(RelationGetRelationName(indrel)); + } +} diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index fcfcf56f4f..1dee4e1ff2 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -582,6 +582,7 @@ EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData EquivalenceClass EquivalenceMember +errcb_phase ErrorContextCallback ErrorData EstimateDSMForeignScan_function -- 2.17.0 --gPQW1Pk7T/0rhUBV Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v27-0002-Save-the-values-of-the-callback.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v16 1/3] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 94 ++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 8ce5011..43859bd 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -292,6 +292,13 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relnamespace; + char *relname; + BlockNumber blkno; + int phase; /* Reusing same enums as for progress reporting */ +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -361,6 +368,7 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); /* @@ -724,6 +732,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -870,6 +880,17 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + errcbarg.relname = relname; + errcbarg.blkno = InvalidBlockNumber; /* Not known yet */ + errcbarg.phase = PROGRESS_VACUUM_PHASE_SCAN_HEAP; + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,6 +912,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -987,6 +1010,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, vmbuffer = InvalidBuffer; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* Work on all the indexes, then the heap */ lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); @@ -994,6 +1020,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, /* Remove tuples from heap */ lazy_vacuum_heap(onerel, vacrelstats); + /* Replace error context while continuing heap scan */ + error_context_stack = &errcallback; + /* * Forget the now-vacuumed tuples, and press on, but be careful * not to reset latestRemovedXid since we want that value to be @@ -1597,6 +1626,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -1772,11 +1804,26 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) int npages; PGRUsage ru0; Buffer vmbuffer = InvalidBuffer; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; /* Report that we are now vacuuming the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + /* + * Setup error traceback support for ereport() + */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + errcbarg.relname = RelationGetRelationName(onerel); + errcbarg.blkno = InvalidBlockNumber; /* Not known yet */ + errcbarg.phase = PROGRESS_VACUUM_PHASE_VACUUM_HEAP; + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + pg_rusage_init(&ru0); npages = 0; @@ -1791,6 +1838,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + errcbarg.blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1811,6 +1859,9 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) npages++; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (BufferIsValid(vmbuffer)) { ReleaseBuffer(vmbuffer); @@ -2318,6 +2369,8 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -2329,10 +2382,23 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(indrel)); + errcbarg.relname = RelationGetRelationName(indrel); + errcbarg.phase = PROGRESS_VACUUM_PHASE_VACUUM_INDEX; + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (IsParallelWorker()) msg = gettext_noop("scanned index \"%s\" to remove %d row versions by parallel vacuum worker"); else @@ -3375,3 +3441,31 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + + switch (cbarg->phase) { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + if (cbarg->blkno!=InvalidBlockNumber) + errcontext(_("while scanning block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + if (cbarg->blkno!=InvalidBlockNumber) + errcontext(_("while vacuuming block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + errcontext(_("while vacuuming index \"%s.%s\""), + cbarg->relnamespace, cbarg->relname); + break; + } +} -- 2.7.4 --Ubca5Sl/zyRO+Ah+ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0002-Include-name-of-table-in-callback-for-index-vacu.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v9 2/3] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 9849685..c96abdf 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -289,6 +289,12 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relname; + char *relnamespace; + BlockNumber blkno; +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -358,6 +364,7 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); /* @@ -803,6 +810,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -879,6 +888,15 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, next_unskippable_block = 0; skipping_blocks = skip_blocks(onerel, params, &next_unskippable_block, nblocks, &vmbuffer, aggressive); + /* Setup error traceback support for ereport() */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + errcbarg.relname = relname; + errcbarg.blkno = InvalidBlockNumber; /* Not known yet */ + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -900,6 +918,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -966,8 +986,11 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, } /* Work on all the indexes, then the heap */ + /* Don't use the errcontext handler outside this function */ + error_context_stack = errcallback.previous; lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); + error_context_stack = &errcallback; /* Remove tuples from heap */ lazy_vacuum_heap(onerel, vacrelstats); @@ -1575,6 +1598,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -3353,3 +3379,14 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); +} -- 2.7.4 --LwgCY5hL6Igino+O Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0003-add-errcontext-callback-in-lazy_vacuum_heap-too.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v6 4/6] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 98a3598..7d6282e 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -138,6 +138,12 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relname; + char *relnamespace; + BlockNumber blkno; +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -176,6 +182,7 @@ static bool lazy_tid_reaped(ItemPointer itemptr, void *state); static int vac_cmp_itemptr(const void *left, const void *right); static bool heap_page_is_all_visible(Relation rel, Buffer buf, TransactionId *visibility_cutoff_xid, bool *all_frozen); +static void vacuum_error_callback(void *arg); /* @@ -607,6 +614,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -648,6 +657,15 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, next_unskippable_block = 0; skipping_blocks = skip_blocks(onerel, params, &next_unskippable_block, nblocks, &vmbuffer, aggressive); + /* Setup error traceback support for ereport() */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + errcbarg.relname = relname; + errcbarg.blkno = InvalidBlockNumber; /* Not known yet */ + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -669,6 +687,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -735,7 +755,10 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, } + /* Don't use the errcontext handler outside this function */ + error_context_stack = errcallback.previous; lazy_vacuum_index_and_heap(onerel, vacrelstats, Irel, nindexes, indstats); + error_context_stack = &errcallback; /* * Forget the now-vacuumed tuples, and press on, but be careful @@ -1340,6 +1363,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -2316,3 +2342,14 @@ heap_page_is_all_visible(Relation rel, Buffer buf, return all_visible; } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); +} -- 2.7.4 --vEao7xgI/oilGqZ+ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v6-0005-add-errcontext-callback-in-lazy_vacuum_heap-too.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v7 4/6] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 4c28876..fb19af4 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -138,6 +138,12 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relname; + char *relnamespace; + BlockNumber blkno; +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -178,6 +184,7 @@ static bool lazy_tid_reaped(ItemPointer itemptr, void *state); static int vac_cmp_itemptr(const void *left, const void *right); static bool heap_page_is_all_visible(Relation rel, Buffer buf, TransactionId *visibility_cutoff_xid, bool *all_frozen); +static void vacuum_error_callback(void *arg); /* @@ -609,6 +616,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -650,6 +659,15 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, next_unskippable_block = 0; skipping_blocks = skip_blocks(onerel, params, &next_unskippable_block, nblocks, &vmbuffer, aggressive); + /* Setup error traceback support for ereport() */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + errcbarg.relname = relname; + errcbarg.blkno = InvalidBlockNumber; /* Not known yet */ + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -671,6 +689,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -737,8 +757,11 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, } /* Work on all the indexes, then the heap */ + /* Don't use the errcontext handler outside this function */ + error_context_stack = errcallback.previous; lazy_vacuum_all_indexes(onerel, vacrelstats, Irel, nindexes, indstats); + error_context_stack = &errcallback; /* Remove tuples from heap */ lazy_vacuum_heap(onerel, vacrelstats); @@ -1346,6 +1369,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -2323,3 +2349,14 @@ heap_page_is_all_visible(Relation rel, Buffer buf, return all_visible; } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); +} -- 2.7.4 --cvVnyQ+4j833TQvp Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v7-0005-add-errcontext-callback-in-lazy_vacuum_heap-too.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v5 4/5] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 145a8a2..5f44543 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -138,6 +138,12 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relname; + char *relnamespace; + BlockNumber blkno; +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -176,6 +182,7 @@ static bool lazy_tid_reaped(ItemPointer itemptr, void *state); static int vac_cmp_itemptr(const void *left, const void *right); static bool heap_page_is_all_visible(Relation rel, Buffer buf, TransactionId *visibility_cutoff_xid, bool *all_frozen); +static void vacuum_error_callback(void *arg); /* @@ -525,6 +532,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -636,6 +645,15 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + errcallback.callback = vacuum_error_callback; + errcbarg.relname = relname; + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + errcbarg.blkno = InvalidBlockNumber; /* Not known yet */ + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -657,6 +675,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -754,7 +774,10 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, } + /* Don't use the errcontext handler outside this function */ + error_context_stack = errcallback.previous; lazy_vacuum_heap_index(onerel, vacrelstats, Irel, nindexes, indstats); + error_context_stack = &errcallback; /* * Forget the now-vacuumed tuples, and press on, but be careful @@ -1359,6 +1382,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -2335,3 +2361,15 @@ heap_page_is_all_visible(Relation rel, Buffer buf, return all_visible; } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); +} -- 2.7.4 --e5GLnnZ8mDMEwH4V Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0005-add-errcontext-callback-in-lazy_vacuum_heap-too.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v28 1/5] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) Discussion: https://www.postgresql.org/message-id/[email protected] --- src/backend/access/heap/vacuumlazy.c | 209 +++++++++++++++++++++++---- src/tools/pgindent/typedefs.list | 1 + 2 files changed, 185 insertions(+), 25 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 03c43efc32..768a69120b 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -268,8 +268,19 @@ typedef struct LVParallelState int nindexes_parallel_condcleanup; } LVParallelState; +typedef enum +{ + VACUUM_ERRCB_PHASE_UNKNOWN, + VACUUM_ERRCB_PHASE_SCAN_HEAP, + VACUUM_ERRCB_PHASE_VACUUM_INDEX, + VACUUM_ERRCB_PHASE_VACUUM_HEAP, + VACUUM_ERRCB_PHASE_INDEX_CLEANUP, +} errcb_phase; + typedef struct LVRelStats { + char *relnamespace; + char *relname; /* useindex = true means two-pass strategy; false means one-pass */ bool useindex; /* Overall statistics about rel */ @@ -290,8 +301,12 @@ typedef struct LVRelStats int num_index_scans; TransactionId latestRemovedXid; bool lock_waiter_detected; -} LVRelStats; + /* Used for error callback: */ + char *indname; + BlockNumber blkno; /* used only for heap operations */ + errcb_phase phase; +} LVRelStats; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -314,10 +329,10 @@ static void lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples); + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats); static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count); + double reltuples, bool estimated_count, LVRelStats *vacrelstats); static int lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, int tupindex, LVRelStats *vacrelstats, Buffer *vmbuffer); static bool should_attempt_truncation(VacuumParams *params, @@ -337,13 +352,13 @@ static void lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult * int nindexes); static void parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVDeadTuples *dead_tuples, - int nindexes); + int nindexes, LVRelStats *vacrelstats); static void vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVSharedIndStats *shared_indstats, - LVDeadTuples *dead_tuples); + LVDeadTuples *dead_tuples, LVRelStats *vacrelstats); static void lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); @@ -361,6 +376,9 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); +static void update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, + BlockNumber blkno, char *indname); /* @@ -460,6 +478,9 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, vacrelstats = (LVRelStats *) palloc0(sizeof(LVRelStats)); + vacrelstats->relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats->relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats->indname = NULL; vacrelstats->old_rel_pages = onerel->rd_rel->relpages; vacrelstats->old_live_tuples = onerel->rd_rel->reltuples; vacrelstats->num_index_scans = 0; @@ -699,7 +720,6 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, BlockNumber nblocks, blkno; HeapTupleData tuple; - char *relname; TransactionId relfrozenxid = onerel->rd_rel->relfrozenxid; TransactionId relminmxid = onerel->rd_rel->relminmxid; BlockNumber empty_pages, @@ -724,20 +744,20 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); - relname = RelationGetRelationName(onerel); if (aggressive) ereport(elevel, (errmsg("aggressively vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); else ereport(elevel, (errmsg("vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); empty_pages = vacuumed_pages = 0; next_fsm_block_to_vacuum = (BlockNumber) 0; @@ -870,6 +890,14 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_SCAN_HEAP, + InvalidBlockNumber, NULL); + errcallback.callback = vacuum_error_callback; + errcallback.arg = vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,8 +919,13 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + vacrelstats->blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_SCAN_HEAP, + blkno, NULL); + if (blkno == next_unskippable_block) { /* Time to advance next_unskippable_block */ @@ -1011,6 +1044,10 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, /* Report that we are once again scanning the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + + /* Set the error context while continuing heap scan */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_SCAN_HEAP, blkno, NULL); } /* @@ -1457,6 +1494,10 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, lazy_vacuum_page(onerel, blkno, buf, 0, vacrelstats, &vmbuffer); vacuumed_pages++; has_dead_tuples = false; + + /* Set the error context while continuing heap scan */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_SCAN_HEAP, blkno, NULL); } else { @@ -1534,7 +1575,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, && VM_ALL_VISIBLE(onerel, blkno, &vmbuffer)) { elog(WARNING, "page is not marked all-visible but visibility map bit is set in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); visibilitymap_clear(onerel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -1555,7 +1596,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else if (PageIsAllVisible(page) && has_dead_tuples) { elog(WARNING, "page containing dead tuples is marked as all-visible in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); PageClearAllVisible(page); MarkBufferDirty(buf); visibilitymap_clear(onerel, blkno, vmbuffer, @@ -1651,6 +1692,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, if (vacrelstats->useindex) lazy_cleanup_all_indexes(Irel, indstats, vacrelstats, lps, nindexes); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* * End parallel mode before updating index statistics as we cannot write * during parallel mode. @@ -1744,7 +1788,7 @@ lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, for (idx = 0; idx < nindexes; idx++) lazy_vacuum_index(Irel[idx], &stats[idx], vacrelstats->dead_tuples, - vacrelstats->old_live_tuples); + vacrelstats->old_live_tuples, vacrelstats); } /* Increase and report the number of index scans */ @@ -1777,6 +1821,10 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_VACUUM_HEAP, + InvalidBlockNumber, NULL); + pg_rusage_init(&ru0); npages = 0; @@ -1791,6 +1839,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + vacrelstats->blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1822,6 +1871,11 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) RelationGetRelationName(onerel), tupindex, npages), errdetail_internal("%s", pg_rusage_show(&ru0)))); + + /* Clear the error traceback phase */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_UNKNOWN, InvalidBlockNumber, + NULL); } /* @@ -1847,6 +1901,10 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_VACUUM_HEAP, + blkno, NULL); + START_CRIT_SECTION(); for (; tupindex < dead_tuples->num_tuples; tupindex++) @@ -1923,6 +1981,11 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, *vmbuffer, visibility_cutoff_xid, flags); } + /* Clear the error traceback phase */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_UNKNOWN, InvalidBlockNumber, + NULL); + return tupindex; } @@ -2083,7 +2146,7 @@ lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, * indexes in the case where no workers are launched. */ parallel_vacuum_index(Irel, stats, lps->lvshared, - vacrelstats->dead_tuples, nindexes); + vacrelstats->dead_tuples, nindexes, vacrelstats); /* Wait for all vacuum workers to finish */ WaitForParallelWorkersToFinish(lps->pcxt); @@ -2106,7 +2169,7 @@ lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, static void parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVDeadTuples *dead_tuples, - int nindexes) + int nindexes, LVRelStats *vacrelstats) { /* * Increment the active worker count if we are able to launch any worker. @@ -2140,7 +2203,7 @@ parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ vacuum_one_index(Irel[idx], &(stats[idx]), lvshared, shared_indstats, - dead_tuples); + dead_tuples, vacrelstats); } /* @@ -2180,7 +2243,8 @@ vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, if (shared_indstats == NULL || skip_parallel_vacuum_index(Irel[i], lps->lvshared)) vacuum_one_index(Irel[i], &(stats[i]), lps->lvshared, - shared_indstats, vacrelstats->dead_tuples); + shared_indstats, vacrelstats->dead_tuples, + vacrelstats); } /* @@ -2200,7 +2264,7 @@ vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, static void vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVSharedIndStats *shared_indstats, - LVDeadTuples *dead_tuples) + LVDeadTuples *dead_tuples, LVRelStats *vacrelstats) { IndexBulkDeleteResult *bulkdelete_res = NULL; @@ -2220,10 +2284,10 @@ vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ if (lvshared->for_cleanup) lazy_cleanup_index(indrel, stats, lvshared->reltuples, - lvshared->estimated_count); + lvshared->estimated_count, vacrelstats); else lazy_vacuum_index(indrel, stats, dead_tuples, - lvshared->reltuples); + lvshared->reltuples, vacrelstats); /* * Copy the index bulk-deletion result returned from ambulkdelete and @@ -2298,7 +2362,8 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, for (idx = 0; idx < nindexes; idx++) lazy_cleanup_index(Irel[idx], &stats[idx], vacrelstats->new_rel_tuples, - vacrelstats->tupcount_pages < vacrelstats->rel_pages); + vacrelstats->tupcount_pages < vacrelstats->rel_pages, + vacrelstats); } } @@ -2313,7 +2378,7 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, */ static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples) + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; @@ -2329,6 +2394,11 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, + RelationGetRelationName(indrel)); + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); @@ -2343,6 +2413,11 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, RelationGetRelationName(indrel), dead_tuples->num_tuples), errdetail_internal("%s", pg_rusage_show(&ru0)))); + + /* Clear the error traceback phase */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_UNKNOWN, InvalidBlockNumber, + NULL); } /* @@ -2354,7 +2429,7 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count) + double reltuples, bool estimated_count, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; @@ -2371,6 +2446,12 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_INDEX_CLEANUP, + InvalidBlockNumber, + RelationGetRelationName(indrel)); + *stats = index_vacuum_cleanup(&ivinfo, *stats); if (!(*stats)) @@ -2392,6 +2473,11 @@ lazy_cleanup_index(Relation indrel, (*stats)->tuples_removed, (*stats)->pages_deleted, (*stats)->pages_free, pg_rusage_show(&ru0)))); + + /* Clear the error traceback phase */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_UNKNOWN, InvalidBlockNumber, + NULL); } /* @@ -3320,6 +3406,8 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) int nindexes; char *sharedquery; IndexBulkDeleteResult **stats; + LVRelStats vacrelstats; + ErrorContextCallback errcallback; lvshared = (LVShared *) shm_toc_lookup(toc, PARALLEL_VACUUM_KEY_SHARED, false); @@ -3369,10 +3457,81 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) if (lvshared->maintenance_work_mem_worker > 0) maintenance_work_mem = lvshared->maintenance_work_mem_worker; + /* Init vacrelstats for use as error callback arg by parallel worker */ + vacrelstats.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats.relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats.indname = NULL; + vacrelstats.phase = VACUUM_ERRCB_PHASE_UNKNOWN; /* Not yet processing */ + + /* Setup error traceback support for ereport() */ + errcallback.callback = vacuum_error_callback; + errcallback.arg = &vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + /* Process indexes to perform vacuum/cleanup */ - parallel_vacuum_index(indrels, stats, lvshared, dead_tuples, nindexes); + parallel_vacuum_index(indrels, stats, lvshared, dead_tuples, nindexes, + &vacrelstats); + + /* Pop the error context stack */ + error_context_stack = errcallback.previous; vac_close_indexes(nindexes, indrels, RowExclusiveLock); table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + LVRelStats *cbarg = arg; + + switch (cbarg->phase) + { + case VACUUM_ERRCB_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while vacuuming block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_VACUUM_INDEX: + errcontext("while vacuuming index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_INDEX_CLEANUP: + errcontext("while cleaning up index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_UNKNOWN: + default: + return; /* do nothing; the cbarg may not be + * initialized */ + } +} + +/* Update vacuum error callback for current phase, block and index */ +static void +update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, BlockNumber blkno, + char *indname) +{ + errcbarg->blkno = blkno; + errcbarg->phase = phase; + + /* Free index name from any previous phase */ + if (errcbarg->indname) + pfree(errcbarg->indname); + + /* For index phases, save the name of the current index for the callback */ + errcbarg->indname = indname ? pstrdup(indname) : NULL; +} diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index fcfcf56f4f..1dee4e1ff2 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -582,6 +582,7 @@ EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData EquivalenceClass EquivalenceMember +errcb_phase ErrorContextCallback ErrorData EstimateDSMForeignScan_function -- 2.17.0 --rG8Oha4Ryswg1dZ8 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v28-0002-Save-the-values-of-the-callback.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v17 1/2] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 117 +++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index a23cdef..9358ab4 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -292,6 +292,14 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relnamespace; + char *relname; + char *indname; /* undefined while not processing index */ + BlockNumber blkno; /* undefined while not processing heap */ + int phase; /* Reusing same enums as for progress reporting */ +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -361,6 +369,7 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); /* @@ -724,6 +733,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -870,6 +881,17 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + errcbarg.relname = relname; + errcbarg.blkno = InvalidBlockNumber; /* Not known yet */ + errcbarg.phase = PROGRESS_VACUUM_PHASE_SCAN_HEAP; + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,6 +913,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -987,6 +1011,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, vmbuffer = InvalidBuffer; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* Work on all the indexes, then the heap */ lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); @@ -1011,6 +1038,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, /* Report that we are once again scanning the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + + /* Set the error context while continuing heap scan */ + error_context_stack = &errcallback; } /* @@ -1597,6 +1627,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -1772,11 +1805,26 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) int npages; PGRUsage ru0; Buffer vmbuffer = InvalidBuffer; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; /* Report that we are now vacuuming the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + /* + * Setup error traceback support for ereport() + */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + errcbarg.relname = RelationGetRelationName(onerel); + errcbarg.blkno = InvalidBlockNumber; /* Not known yet */ + errcbarg.phase = PROGRESS_VACUUM_PHASE_VACUUM_HEAP; + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + pg_rusage_init(&ru0); npages = 0; @@ -1791,6 +1839,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + errcbarg.blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1811,6 +1860,9 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) npages++; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (BufferIsValid(vmbuffer)) { ReleaseBuffer(vmbuffer); @@ -2318,6 +2370,8 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -2329,10 +2383,24 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(indrel)); + errcbarg.indname = RelationGetRelationName(indrel); + errcbarg.relname = get_rel_name(indrel->rd_index->indexrelid); + errcbarg.phase = PROGRESS_VACUUM_PHASE_VACUUM_INDEX; + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (IsParallelWorker()) msg = gettext_noop("scanned index \"%s\" to remove %d row versions by parallel vacuum worker"); else @@ -2359,6 +2427,8 @@ lazy_cleanup_index(Relation indrel, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + vacuum_error_callback_arg errcbarg; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); @@ -2371,8 +2441,22 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(indrel)); + errcbarg.indname = RelationGetRelationName(indrel); + errcbarg.relname = get_rel_name(indrel->rd_index->indexrelid); + errcbarg.phase = PROGRESS_VACUUM_PHASE_INDEX_CLEANUP; + + errcallback.previous = error_context_stack; + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) &errcbarg; + error_context_stack = &errcallback; + *stats = index_vacuum_cleanup(&ivinfo, *stats); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (!(*stats)) return; @@ -3375,3 +3459,36 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + + switch (cbarg->phase) { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext(_("while scanning block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext(_("while vacuuming block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + errcontext(_("while vacuuming index \"%s.%s\" of relation \"%s\""), + cbarg->relnamespace, cbarg->indname, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + errcontext(_("while cleaning up index \"%s.%s\" of relation \"%s\""), + cbarg->relnamespace, cbarg->indname, cbarg->relname); + break; + } +} -- 2.7.4 --byLs0wutDcxFdwtm Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0002-Functions-to-initialize-errcontext.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v18] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) Discussion: https://www.postgresql.org/message-id/[email protected] --- src/backend/access/heap/vacuumlazy.c | 120 +++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index a23cdef..209f483 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -292,6 +292,14 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relnamespace; + char *relname; + char *indname; /* undefined while not processing index */ + BlockNumber blkno; /* undefined while not processing heap */ + int phase; /* Reusing same enums as for progress reporting */ +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -361,6 +369,9 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); +static void init_error_context(ErrorContextCallback *errcallback, + vacuum_error_callback_arg *errcbarg, Relation onerel, int phase); /* @@ -724,6 +735,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -870,6 +883,10 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + init_error_context(&errcallback, &errcbarg, onerel, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,6 +908,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -987,6 +1006,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, vmbuffer = InvalidBuffer; } + /* Pop the error context stack while calling vacuum */ + error_context_stack = errcallback.previous; + /* Work on all the indexes, then the heap */ lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); @@ -1011,6 +1033,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, /* Report that we are once again scanning the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + + /* Set the error context while continuing heap scan */ + error_context_stack = &errcallback; } /* @@ -1597,6 +1622,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -1772,11 +1800,19 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) int npages; PGRUsage ru0; Buffer vmbuffer = InvalidBuffer; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; /* Report that we are now vacuuming the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + /* + * Setup error traceback support for ereport() + */ + init_error_context(&errcallback, &errcbarg, onerel, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + error_context_stack = &errcallback; + pg_rusage_init(&ru0); npages = 0; @@ -1791,6 +1827,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + errcbarg.blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1811,6 +1848,9 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) npages++; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (BufferIsValid(vmbuffer)) { ReleaseBuffer(vmbuffer); @@ -2318,6 +2358,8 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -2329,10 +2371,17 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + init_error_context(&errcallback, &errcbarg, indrel, PROGRESS_VACUUM_PHASE_VACUUM_INDEX); + error_context_stack = &errcallback; + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (IsParallelWorker()) msg = gettext_noop("scanned index \"%s\" to remove %d row versions by parallel vacuum worker"); else @@ -2359,6 +2408,8 @@ lazy_cleanup_index(Relation indrel, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + vacuum_error_callback_arg errcbarg; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); @@ -2371,8 +2422,15 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + init_error_context(&errcallback, &errcbarg, indrel, PROGRESS_VACUUM_PHASE_INDEX_CLEANUP); + error_context_stack = &errcallback; + *stats = index_vacuum_cleanup(&ivinfo, *stats); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (!(*stats)) return; @@ -3375,3 +3433,65 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + + switch (cbarg->phase) { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext(_("while scanning block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext(_("while vacuuming block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + errcontext(_("while vacuuming index \"%s.%s\" of relation \"%s\""), + cbarg->relnamespace, cbarg->indname, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + errcontext(_("while cleaning up index \"%s.%s\" of relation \"%s\""), + cbarg->relnamespace, cbarg->indname, cbarg->relname); + break; + } +} + +/* Initialize error context for heap operations */ +static void +init_error_context(ErrorContextCallback *errcallback, vacuum_error_callback_arg *errcbarg, Relation rel, int phase) +{ + switch (phase) + { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + errcbarg->relname = RelationGetRelationName(rel); + errcbarg->indname = NULL; /* Not used for heap */ + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + /* indname is the index being processed, relname is its relation */ + errcbarg->indname = RelationGetRelationName(rel); + errcbarg->relname = get_rel_name(rel->rd_index->indexrelid); + break; + } + + errcbarg->relnamespace = get_namespace_name(RelationGetNamespace(rel)); + errcbarg->blkno = InvalidBlockNumber; /* Not known yet */ + errcbarg->phase = phase; + + errcallback->callback = vacuum_error_callback; + errcallback->arg = errcbarg; + errcallback->previous = error_context_stack; +} -- 2.7.4 --2fHTh5uZTiUOsy+g-- ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v20 1/2] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) Discussion: https://www.postgresql.org/message-id/[email protected] --- src/backend/access/heap/vacuumlazy.c | 130 ++++++++++++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 1 deletion(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index a23cdef..ce3efd7 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -292,6 +292,14 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relnamespace; + char *relname; + char *indname; + BlockNumber blkno; /* used only for heap operations */ + int phase; /* Reusing same enums as for progress reporting */ +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -361,6 +369,9 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); +static void init_vacuum_error_callback(ErrorContextCallback *errcallback, + vacuum_error_callback_arg *errcbarg, Relation onerel, int phase); /* @@ -724,6 +735,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -870,6 +883,11 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, onerel, + PROGRESS_VACUUM_PHASE_SCAN_HEAP); + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,6 +909,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -987,6 +1007,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, vmbuffer = InvalidBuffer; } + /* Pop the error context stack while calling vacuum */ + error_context_stack = errcallback.previous; + /* Work on all the indexes, then the heap */ lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); @@ -1011,6 +1034,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, /* Report that we are once again scanning the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + + /* Set the error context while continuing heap scan */ + error_context_stack = &errcallback; } /* @@ -1597,6 +1623,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -1772,14 +1801,21 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) int npages; PGRUsage ru0; Buffer vmbuffer = InvalidBuffer; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; /* Report that we are now vacuuming the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); pg_rusage_init(&ru0); - npages = 0; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, onerel, + PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + error_context_stack = &errcallback; + + npages = 0; tupindex = 0; while (tupindex < vacrelstats->dead_tuples->num_tuples) { @@ -1791,6 +1827,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + errcbarg.blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1811,6 +1848,9 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) npages++; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (BufferIsValid(vmbuffer)) { ReleaseBuffer(vmbuffer); @@ -2318,6 +2358,8 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -2329,10 +2371,18 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, indrel, + PROGRESS_VACUUM_PHASE_VACUUM_INDEX); + error_context_stack = &errcallback; + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (IsParallelWorker()) msg = gettext_noop("scanned index \"%s\" to remove %d row versions by parallel vacuum worker"); else @@ -2359,6 +2409,8 @@ lazy_cleanup_index(Relation indrel, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + vacuum_error_callback_arg errcbarg; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); @@ -2371,8 +2423,16 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, indrel, + PROGRESS_VACUUM_PHASE_INDEX_CLEANUP); + error_context_stack = &errcallback; + *stats = index_vacuum_cleanup(&ivinfo, *stats); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (!(*stats)) return; @@ -3375,3 +3435,71 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + + switch (cbarg->phase) { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext(_("while scanning block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext(_("while vacuuming block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + errcontext(_("while vacuuming index \"%s.%s\" of relation \"%s\""), + cbarg->relnamespace, cbarg->indname, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + errcontext(_("while cleaning up index \"%s.%s\" of relation \"%s\""), + cbarg->relnamespace, cbarg->indname, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_TRUNCATE: + case PROGRESS_VACUUM_PHASE_FINAL_CLEANUP: + default: + return; /* Shouldn't happen: do nothing */ + } +} + +/* Initialize vacuum error callback */ +static void +init_vacuum_error_callback(ErrorContextCallback *errcallback, + vacuum_error_callback_arg *errcbarg, Relation rel, int phase) +{ + switch (phase) + { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + errcbarg->relname = RelationGetRelationName(rel); + errcbarg->indname = NULL; /* Not used for heap */ + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + /* rel is an index relation in index vacuum case */ + errcbarg->relname = get_rel_name(rel->rd_index->indrelid); + errcbarg->indname = RelationGetRelationName(rel); + break; + } + + errcbarg->relnamespace = get_namespace_name(RelationGetNamespace(rel)); + errcbarg->blkno = InvalidBlockNumber; /* Not known yet */ + errcbarg->phase = phase; + + errcallback->callback = vacuum_error_callback; + errcallback->arg = errcbarg; + errcallback->previous = error_context_stack; +} -- 2.7.4 --Wtrm9ATX0sn6fFKv Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v20-0002-add-callback-for-truncation.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v24 1/4] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) Discussion: https://www.postgresql.org/message-id/[email protected] --- src/backend/access/heap/vacuumlazy.c | 192 +++++++++++++++++++++++---- 1 file changed, 167 insertions(+), 25 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 03c43efc32..084b43c178 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -268,8 +268,19 @@ typedef struct LVParallelState int nindexes_parallel_condcleanup; } LVParallelState; +typedef enum { + VACUUM_ERRCB_PHASE_UNKNOWN, + VACUUM_ERRCB_PHASE_SCAN_HEAP, + VACUUM_ERRCB_PHASE_VACUUM_INDEX, + VACUUM_ERRCB_PHASE_VACUUM_HEAP, + VACUUM_ERRCB_PHASE_INDEX_CLEANUP, + VACUUM_ERRCB_PHASE_VACUUM_FSM, +} errcb_phase; + typedef struct LVRelStats { + char *relnamespace; + char *relname; /* useindex = true means two-pass strategy; false means one-pass */ bool useindex; /* Overall statistics about rel */ @@ -290,8 +301,12 @@ typedef struct LVRelStats int num_index_scans; TransactionId latestRemovedXid; bool lock_waiter_detected; -} LVRelStats; + /* Used for error callback: */ + char *indname; + BlockNumber blkno; /* used only for heap operations */ + errcb_phase phase; +} LVRelStats; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -314,10 +329,10 @@ static void lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples); + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats); static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count); + double reltuples, bool estimated_count, LVRelStats *vacrelstats); static int lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, int tupindex, LVRelStats *vacrelstats, Buffer *vmbuffer); static bool should_attempt_truncation(VacuumParams *params, @@ -337,13 +352,13 @@ static void lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult * int nindexes); static void parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVDeadTuples *dead_tuples, - int nindexes); + int nindexes, LVRelStats *vacrelstats); static void vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); static void vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVSharedIndStats *shared_indstats, - LVDeadTuples *dead_tuples); + LVDeadTuples *dead_tuples, LVRelStats *vacrelstats); static void lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, LVRelStats *vacrelstats, LVParallelState *lps, int nindexes); @@ -361,6 +376,9 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); +static void update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, + BlockNumber blkno, Relation rel); /* @@ -460,6 +478,9 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, vacrelstats = (LVRelStats *) palloc0(sizeof(LVRelStats)); + vacrelstats->relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats->relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats->indname = NULL; vacrelstats->old_rel_pages = onerel->rd_rel->relpages; vacrelstats->old_live_tuples = onerel->rd_rel->reltuples; vacrelstats->num_index_scans = 0; @@ -699,7 +720,6 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, BlockNumber nblocks, blkno; HeapTupleData tuple; - char *relname; TransactionId relfrozenxid = onerel->rd_rel->relfrozenxid; TransactionId relminmxid = onerel->rd_rel->relminmxid; BlockNumber empty_pages, @@ -724,20 +744,20 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); - relname = RelationGetRelationName(onerel); if (aggressive) ereport(elevel, (errmsg("aggressively vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); else ereport(elevel, (errmsg("vacuuming \"%s.%s\"", - get_namespace_name(RelationGetNamespace(onerel)), - relname))); + vacrelstats->relnamespace, + vacrelstats->relname))); empty_pages = vacuumed_pages = 0; next_fsm_block_to_vacuum = (BlockNumber) 0; @@ -870,6 +890,14 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_SCAN_HEAP, + InvalidBlockNumber, NULL); + errcallback.callback = vacuum_error_callback; + errcallback.arg = vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,6 +919,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + vacrelstats->blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -1005,12 +1035,18 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, * Vacuum the Free Space Map to make newly-freed space visible on * upper-level FSM pages. Note we have not yet processed blkno. */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_VACUUM_FSM, InvalidBlockNumber, NULL); FreeSpaceMapVacuumRange(onerel, next_fsm_block_to_vacuum, blkno); next_fsm_block_to_vacuum = blkno; /* Report that we are once again scanning the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + + /* Set the error context while continuing heap scan */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_SCAN_HEAP, blkno, NULL); } /* @@ -1488,9 +1524,15 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, */ if (blkno - next_fsm_block_to_vacuum >= VACUUM_FSM_EVERY_PAGES) { + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_VACUUM_FSM, InvalidBlockNumber, + NULL); FreeSpaceMapVacuumRange(onerel, next_fsm_block_to_vacuum, blkno); next_fsm_block_to_vacuum = blkno; + /* Set the error context while continuing heap scan */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_SCAN_HEAP, blkno, NULL); } } @@ -1534,7 +1576,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, && VM_ALL_VISIBLE(onerel, blkno, &vmbuffer)) { elog(WARNING, "page is not marked all-visible but visibility map bit is set in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); visibilitymap_clear(onerel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -1555,7 +1597,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else if (PageIsAllVisible(page) && has_dead_tuples) { elog(WARNING, "page containing dead tuples is marked as all-visible in relation \"%s\" page %u", - relname, blkno); + vacrelstats->relname, blkno); PageClearAllVisible(page); MarkBufferDirty(buf); visibilitymap_clear(onerel, blkno, vmbuffer, @@ -1642,7 +1684,11 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, * not there were indexes. */ if (blkno > next_fsm_block_to_vacuum) + { + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_VACUUM_FSM, InvalidBlockNumber, NULL); FreeSpaceMapVacuumRange(onerel, next_fsm_block_to_vacuum, blkno); + } /* report all blocks vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); @@ -1651,6 +1697,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, if (vacrelstats->useindex) lazy_cleanup_all_indexes(Irel, indstats, vacrelstats, lps, nindexes); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* * End parallel mode before updating index statistics as we cannot write * during parallel mode. @@ -1744,7 +1793,7 @@ lazy_vacuum_all_indexes(Relation onerel, Relation *Irel, for (idx = 0; idx < nindexes; idx++) lazy_vacuum_index(Irel[idx], &stats[idx], vacrelstats->dead_tuples, - vacrelstats->old_live_tuples); + vacrelstats->old_live_tuples, vacrelstats); } /* Increase and report the number of index scans */ @@ -1847,6 +1896,10 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer, pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno); + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, VACUUM_ERRCB_PHASE_VACUUM_HEAP, + blkno, NULL); + START_CRIT_SECTION(); for (; tupindex < dead_tuples->num_tuples; tupindex++) @@ -2083,7 +2136,7 @@ lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, * indexes in the case where no workers are launched. */ parallel_vacuum_index(Irel, stats, lps->lvshared, - vacrelstats->dead_tuples, nindexes); + vacrelstats->dead_tuples, nindexes, vacrelstats); /* Wait for all vacuum workers to finish */ WaitForParallelWorkersToFinish(lps->pcxt); @@ -2106,7 +2159,7 @@ lazy_parallel_vacuum_indexes(Relation *Irel, IndexBulkDeleteResult **stats, static void parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVDeadTuples *dead_tuples, - int nindexes) + int nindexes, LVRelStats *vacrelstats) { /* * Increment the active worker count if we are able to launch any worker. @@ -2140,7 +2193,7 @@ parallel_vacuum_index(Relation *Irel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ vacuum_one_index(Irel[idx], &(stats[idx]), lvshared, shared_indstats, - dead_tuples); + dead_tuples, vacrelstats); } /* @@ -2180,7 +2233,8 @@ vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, if (shared_indstats == NULL || skip_parallel_vacuum_index(Irel[i], lps->lvshared)) vacuum_one_index(Irel[i], &(stats[i]), lps->lvshared, - shared_indstats, vacrelstats->dead_tuples); + shared_indstats, vacrelstats->dead_tuples, + vacrelstats); } /* @@ -2200,7 +2254,7 @@ vacuum_indexes_leader(Relation *Irel, IndexBulkDeleteResult **stats, static void vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, LVShared *lvshared, LVSharedIndStats *shared_indstats, - LVDeadTuples *dead_tuples) + LVDeadTuples *dead_tuples, LVRelStats *vacrelstats) { IndexBulkDeleteResult *bulkdelete_res = NULL; @@ -2220,10 +2274,10 @@ vacuum_one_index(Relation indrel, IndexBulkDeleteResult **stats, /* Do vacuum or cleanup of the index */ if (lvshared->for_cleanup) lazy_cleanup_index(indrel, stats, lvshared->reltuples, - lvshared->estimated_count); + lvshared->estimated_count, vacrelstats); else lazy_vacuum_index(indrel, stats, dead_tuples, - lvshared->reltuples); + lvshared->reltuples, vacrelstats); /* * Copy the index bulk-deletion result returned from ambulkdelete and @@ -2298,7 +2352,8 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, for (idx = 0; idx < nindexes; idx++) lazy_cleanup_index(Irel[idx], &stats[idx], vacrelstats->new_rel_tuples, - vacrelstats->tupcount_pages < vacrelstats->rel_pages); + vacrelstats->tupcount_pages < vacrelstats->rel_pages, + vacrelstats); } } @@ -2313,7 +2368,7 @@ lazy_cleanup_all_indexes(Relation *Irel, IndexBulkDeleteResult **stats, */ static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, - LVDeadTuples *dead_tuples, double reltuples) + LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; @@ -2329,6 +2384,11 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_VACUUM_INDEX, InvalidBlockNumber, + indrel); + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); @@ -2354,7 +2414,7 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, static void lazy_cleanup_index(Relation indrel, IndexBulkDeleteResult **stats, - double reltuples, bool estimated_count) + double reltuples, bool estimated_count, LVRelStats *vacrelstats) { IndexVacuumInfo ivinfo; const char *msg; @@ -2371,6 +2431,10 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + update_vacuum_error_cbarg(vacrelstats, + VACUUM_ERRCB_PHASE_INDEX_CLEANUP, InvalidBlockNumber, indrel); + *stats = index_vacuum_cleanup(&ivinfo, *stats); if (!(*stats)) @@ -3320,6 +3384,8 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) int nindexes; char *sharedquery; IndexBulkDeleteResult **stats; + LVRelStats vacrelstats; + ErrorContextCallback errcallback; lvshared = (LVShared *) shm_toc_lookup(toc, PARALLEL_VACUUM_KEY_SHARED, false); @@ -3341,6 +3407,18 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) */ onerel = table_open(lvshared->relid, ShareUpdateExclusiveLock); + /* Init vacrelstats for use as error callback by parallel worker: */ + vacrelstats.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats.relname = pstrdup(RelationGetRelationName(onerel)); + vacrelstats.indname = NULL; + vacrelstats.phase = VACUUM_ERRCB_PHASE_UNKNOWN; /* Not yet processing */ + + /* Setup error traceback support for ereport() */ + errcallback.callback = vacuum_error_callback; + errcallback.arg = &vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + /* * Open all indexes. indrels are sorted in order by OID, which should be * matched to the leader's one. @@ -3370,9 +3448,73 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) maintenance_work_mem = lvshared->maintenance_work_mem_worker; /* Process indexes to perform vacuum/cleanup */ - parallel_vacuum_index(indrels, stats, lvshared, dead_tuples, nindexes); + parallel_vacuum_index(indrels, stats, lvshared, dead_tuples, nindexes, + &vacrelstats); vac_close_indexes(nindexes, indrels, RowExclusiveLock); table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + LVRelStats *cbarg = arg; + + switch (cbarg->phase) { + case VACUUM_ERRCB_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext("while vacuuming block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_VACUUM_INDEX: + errcontext("while vacuuming index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_INDEX_CLEANUP: + errcontext("while cleaning up index \"%s\" of relation \"%s.%s\"", + cbarg->indname, cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_VACUUM_FSM: + errcontext("while vacuuming free space map of relation \"%s.%s\"", + cbarg->relnamespace, cbarg->relname); + break; + + case VACUUM_ERRCB_PHASE_UNKNOWN: + default: + return; /* do nothing; the cbarg maybe isn't yet initialized */ + } +} + +/* Update vacuum error callback for current phase, block and index */ +static void +update_vacuum_error_cbarg(LVRelStats *errcbarg, int phase, BlockNumber blkno, + Relation indrel) +{ + errcbarg->blkno = blkno; + errcbarg->phase = phase; + + /* Free index name from any previous phase */ + if (errcbarg->indname) { + pfree(errcbarg->indname); + errcbarg->indname = NULL; + } + + /* For index phases, save the name of the current index for the callback */ + if (indrel) { + Assert(indrel->rd_rel->relkind == RELKIND_INDEX); + errcbarg->indname = pstrdup(RelationGetRelationName(indrel)); + } +} -- 2.17.0 --a8Wt8u1KmwUX3Y2C Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v24-0002-Drop-reltuples.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v11 1/6] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 38 ++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 1dc5294..985293b 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -290,8 +290,12 @@ typedef struct LVRelStats int num_index_scans; TransactionId latestRemovedXid; bool lock_waiter_detected; -} LVRelStats; + /* Used by the error callback */ + char *relname; + char *relnamespace; + BlockNumber blkno; +} LVRelStats; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -361,6 +365,7 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); /* @@ -813,6 +818,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); @@ -892,6 +898,16 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, next_unskippable_block = 0; skipping_blocks = skip_blocks(onerel, params, &next_unskippable_block, nblocks, &vmbuffer, aggressive); + /* Setup error traceback support for ereport() */ + vacrelstats->relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + vacrelstats->relname = relname; + vacrelstats->blkno = InvalidBlockNumber; /* Not known yet */ + + errcallback.callback = vacuum_error_callback; + errcallback.arg = (void *) vacrelstats; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -913,6 +929,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + vacrelstats->blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -979,11 +997,13 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, } /* Work on all the indexes, then the heap */ + /* Don't use the errcontext handler outside this function */ + error_context_stack = errcallback.previous; lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); - /* Remove tuples from heap */ lazy_vacuum_heap(onerel, vacrelstats); + error_context_stack = &errcallback; /* * Forget the now-vacuumed tuples, and press on, but be careful @@ -1602,6 +1622,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -3416,3 +3439,14 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + LVRelStats *cbarg = arg; + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); +} -- 2.7.4 --5vNYLRcllDrimb99 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v11-0002-add-errcontext-callback-in-lazy_vacuum_heap-too.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v4 4/5] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) As requested here. https://www.postgresql.org/message-id/20190807235154.erbmr4o4bo6vgnjv%40alap3.anarazel.de --- src/backend/access/heap/vacuumlazy.c | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 07f86c7..d5d2b69 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -138,6 +138,12 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relname; + char *relnamespace; + BlockNumber blkno; +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -176,6 +182,7 @@ static bool lazy_tid_reaped(ItemPointer itemptr, void *state); static int vac_cmp_itemptr(const void *left, const void *right); static bool heap_page_is_all_visible(Relation rel, Buffer buf, TransactionId *visibility_cutoff_xid, bool *all_frozen); +static void vacuum_error_callback(void *arg); /* @@ -525,6 +532,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -636,6 +645,15 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + errcallback.callback = vacuum_error_callback; + errcbarg.relname = relname; + errcbarg.relnamespace = get_namespace_name(RelationGetNamespace(onerel)); + errcbarg.blkno = InvalidBlockNumber; /* Not known yet */ + errcallback.arg = (void *) &errcbarg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -657,6 +675,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -754,7 +774,11 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, } + /* Don't use the errcontext handler outside this function */ + error_context_stack = errcallback.previous; lazy_vacuum_heap_index(onerel, vacrelstats, Irel, nindexes, indstats); + + error_context_stack = &errcallback; vacrelstats->num_dead_tuples = 0; /* @@ -1353,6 +1377,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -2334,3 +2361,15 @@ heap_page_is_all_visible(Relation rel, Buffer buf, return all_visible; } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + + errcontext("while scanning block %u of relation \"%s.%s\"", + cbarg->blkno, cbarg->relnamespace, cbarg->relname); +} -- 2.7.4 --zhtSGe8h3+lMyY1M Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0005-add-errcontext-callback-in-lazy_vacuum_heap-too.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v19] vacuum errcontext to show block being processed @ 2019-12-13 02:54 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Justin Pryzby @ 2019-12-13 02:54 UTC (permalink / raw) Discussion: https://www.postgresql.org/message-id/[email protected] --- src/backend/access/heap/vacuumlazy.c | 120 +++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index a23cdef..ebfb2e7 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -292,6 +292,14 @@ typedef struct LVRelStats bool lock_waiter_detected; } LVRelStats; +typedef struct +{ + char *relnamespace; + char *relname; + char *indname; + BlockNumber blkno; /* used only for heap operations */ + int phase; /* Reusing same enums as for progress reporting */ +} vacuum_error_callback_arg; /* A few variables that don't seem worth passing around as parameters */ static int elevel = -1; @@ -361,6 +369,9 @@ static void end_parallel_vacuum(Relation *Irel, IndexBulkDeleteResult **stats, LVParallelState *lps, int nindexes); static LVSharedIndStats *get_indstats(LVShared *lvshared, int n); static bool skip_parallel_vacuum_index(Relation indrel, LVShared *lvshared); +static void vacuum_error_callback(void *arg); +static void init_vacuum_error_callback(ErrorContextCallback *errcallback, + vacuum_error_callback_arg *errcbarg, Relation onerel, int phase); /* @@ -724,6 +735,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, PROGRESS_VACUUM_MAX_DEAD_TUPLES }; int64 initprog_val[3]; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -870,6 +883,10 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, else skipping_blocks = false; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, onerel, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + error_context_stack = &errcallback; + for (blkno = 0; blkno < nblocks; blkno++) { Buffer buf; @@ -891,6 +908,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, #define FORCE_CHECK_PAGE() \ (blkno == nblocks - 1 && should_attempt_truncation(params, vacrelstats)) + errcbarg.blkno = blkno; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); if (blkno == next_unskippable_block) @@ -987,6 +1006,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, vmbuffer = InvalidBuffer; } + /* Pop the error context stack while calling vacuum */ + error_context_stack = errcallback.previous; + /* Work on all the indexes, then the heap */ lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats, lps, nindexes); @@ -1011,6 +1033,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, /* Report that we are once again scanning the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_SCAN_HEAP); + + /* Set the error context while continuing heap scan */ + error_context_stack = &errcallback; } /* @@ -1597,6 +1622,9 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats, RecordPageWithFreeSpace(onerel, blkno, freespace); } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + /* report that everything is scanned and vacuumed */ pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, blkno); @@ -1772,11 +1800,19 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) int npages; PGRUsage ru0; Buffer vmbuffer = InvalidBuffer; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; /* Report that we are now vacuuming the heap */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + /* + * Setup error traceback support for ereport() + */ + init_vacuum_error_callback(&errcallback, &errcbarg, onerel, PROGRESS_VACUUM_PHASE_VACUUM_HEAP); + error_context_stack = &errcallback; + pg_rusage_init(&ru0); npages = 0; @@ -1791,6 +1827,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) vacuum_delay_point(); tblk = ItemPointerGetBlockNumber(&vacrelstats->dead_tuples->itemptrs[tupindex]); + errcbarg.blkno = tblk; buf = ReadBufferExtended(onerel, MAIN_FORKNUM, tblk, RBM_NORMAL, vac_strategy); if (!ConditionalLockBufferForCleanup(buf)) @@ -1811,6 +1848,9 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats) npages++; } + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (BufferIsValid(vmbuffer)) { ReleaseBuffer(vmbuffer); @@ -2318,6 +2358,8 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + ErrorContextCallback errcallback; + vacuum_error_callback_arg errcbarg; pg_rusage_init(&ru0); @@ -2329,10 +2371,17 @@ lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, indrel, PROGRESS_VACUUM_PHASE_VACUUM_INDEX); + error_context_stack = &errcallback; + /* Do bulk deletion */ *stats = index_bulk_delete(&ivinfo, *stats, lazy_tid_reaped, (void *) dead_tuples); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (IsParallelWorker()) msg = gettext_noop("scanned index \"%s\" to remove %d row versions by parallel vacuum worker"); else @@ -2359,6 +2408,8 @@ lazy_cleanup_index(Relation indrel, IndexVacuumInfo ivinfo; const char *msg; PGRUsage ru0; + vacuum_error_callback_arg errcbarg; + ErrorContextCallback errcallback; pg_rusage_init(&ru0); @@ -2371,8 +2422,15 @@ lazy_cleanup_index(Relation indrel, ivinfo.num_heap_tuples = reltuples; ivinfo.strategy = vac_strategy; + /* Setup error traceback support for ereport() */ + init_vacuum_error_callback(&errcallback, &errcbarg, indrel, PROGRESS_VACUUM_PHASE_INDEX_CLEANUP); + error_context_stack = &errcallback; + *stats = index_vacuum_cleanup(&ivinfo, *stats); + /* Pop the error context stack */ + error_context_stack = errcallback.previous; + if (!(*stats)) return; @@ -3375,3 +3433,65 @@ parallel_vacuum_main(dsm_segment *seg, shm_toc *toc) table_close(onerel, ShareUpdateExclusiveLock); pfree(stats); } + +/* + * Error context callback for errors occurring during vacuum. + */ +static void +vacuum_error_callback(void *arg) +{ + vacuum_error_callback_arg *cbarg = arg; + + switch (cbarg->phase) { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext(_("while scanning block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + if (BlockNumberIsValid(cbarg->blkno)) + errcontext(_("while vacuuming block %u of relation \"%s.%s\""), + cbarg->blkno, cbarg->relnamespace, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + errcontext(_("while vacuuming index \"%s.%s\" of relation \"%s\""), + cbarg->relnamespace, cbarg->indname, cbarg->relname); + break; + + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + errcontext(_("while cleaning up index \"%s.%s\" of relation \"%s\""), + cbarg->relnamespace, cbarg->indname, cbarg->relname); + break; + } +} + +/* Initialize vacuum error callback */ +static void +init_vacuum_error_callback(ErrorContextCallback *errcallback, vacuum_error_callback_arg *errcbarg, Relation rel, int phase) +{ + switch (phase) + { + case PROGRESS_VACUUM_PHASE_SCAN_HEAP: + case PROGRESS_VACUUM_PHASE_VACUUM_HEAP: + errcbarg->relname = RelationGetRelationName(rel); + errcbarg->indname = NULL; /* Not used for heap */ + break; + + case PROGRESS_VACUUM_PHASE_VACUUM_INDEX: + case PROGRESS_VACUUM_PHASE_INDEX_CLEANUP: + /* rel is an index relation in index vacuum case */ + errcbarg->relname = get_rel_name(rel->rd_index->indrelid); + errcbarg->indname = RelationGetRelationName(rel); + break; + } + + errcbarg->relnamespace = get_namespace_name(RelationGetNamespace(rel)); + errcbarg->blkno = InvalidBlockNumber; /* Not known yet */ + errcbarg->phase = phase; + + errcallback->callback = vacuum_error_callback; + errcallback->arg = errcbarg; + errcallback->previous = error_context_stack; +} -- 2.7.4 --E/DnYTRukya0zdZ1-- ^ permalink raw reply [nested|flat] 30+ messages in thread
* [PATCH v1 1/2] Add CREATE OR REPLACE MATERIALIZED VIEW @ 2024-05-21 16:35 Erik Wienhold <[email protected]> 0 siblings, 0 replies; 30+ messages in thread From: Erik Wienhold @ 2024-05-21 16:35 UTC (permalink / raw) --- .../sgml/ref/create_materialized_view.sgml | 15 +- src/backend/commands/createas.c | 207 ++++++++++++++---- src/backend/commands/tablecmds.c | 8 +- src/backend/commands/view.c | 106 ++++++--- src/backend/parser/gram.y | 15 ++ src/bin/psql/tab-complete.c | 2 +- src/include/commands/view.h | 3 + src/include/nodes/parsenodes.h | 2 +- src/include/nodes/primnodes.h | 1 + src/test/regress/expected/matview.out | 191 ++++++++++++++++ src/test/regress/sql/matview.sql | 108 +++++++++ 11 files changed, 574 insertions(+), 84 deletions(-) diff --git a/doc/src/sgml/ref/create_materialized_view.sgml b/doc/src/sgml/ref/create_materialized_view.sgml index 0d2fea2b97..b5a8e3441a 100644 --- a/doc/src/sgml/ref/create_materialized_view.sgml +++ b/doc/src/sgml/ref/create_materialized_view.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation <refsynopsisdiv> <synopsis> -CREATE MATERIALIZED VIEW [ IF NOT EXISTS ] <replaceable>table_name</replaceable> +CREATE [ OR REPLACE ] MATERIALIZED VIEW [ IF NOT EXISTS ] <replaceable>table_name</replaceable> [ (<replaceable>column_name</replaceable> [, ...] ) ] [ USING <replaceable class="parameter">method</replaceable> ] [ WITH ( <replaceable class="parameter">storage_parameter</replaceable> [= <replaceable class="parameter">value</replaceable>] [, ... ] ) ] @@ -60,6 +60,17 @@ CREATE MATERIALIZED VIEW [ IF NOT EXISTS ] <replaceable>table_name</replaceable> <title>Parameters</title> <variablelist> + <varlistentry> + <term><literal>OR REPLACE</literal></term> + <listitem> + <para> + Replaces a materialized view if it already exists. + Specifying <literal>OR REPLACE</literal> together with + <literal>IF NOT EXISTS</literal> is an error. + </para> + </listitem> + </varlistentry> + <varlistentry> <term><literal>IF NOT EXISTS</literal></term> <listitem> @@ -67,7 +78,7 @@ CREATE MATERIALIZED VIEW [ IF NOT EXISTS ] <replaceable>table_name</replaceable> Do not throw an error if a materialized view with the same name already exists. A notice is issued in this case. Note that there is no guarantee that the existing materialized view is anything like the one that would - have been created. + have been created, unless you use <literal>OR REPLACE</literal> instead. </para> </listitem> </varlistentry> diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 62050f4dc5..1f34665521 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -79,55 +79,151 @@ static void intorel_destroy(DestReceiver *self); static ObjectAddress create_ctas_internal(List *attrList, IntoClause *into) { - CreateStmt *create = makeNode(CreateStmt); - bool is_matview; + bool is_matview, + replace = false; char relkind; - Datum toast_options; - static char *validnsps[] = HEAP_RELOPT_NAMESPACES; + Oid matviewOid = InvalidOid; ObjectAddress intoRelationAddr; /* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */ is_matview = (into->viewQuery != NULL); relkind = is_matview ? RELKIND_MATVIEW : RELKIND_RELATION; - /* - * Create the target relation by faking up a CREATE TABLE parsetree and - * passing it to DefineRelation. - */ - create->relation = into->rel; - create->tableElts = attrList; - create->inhRelations = NIL; - create->ofTypename = NULL; - create->constraints = NIL; - create->options = into->options; - create->oncommit = into->onCommit; - create->tablespacename = into->tableSpaceName; - create->if_not_exists = false; - create->accessMethod = into->accessMethod; + /* Check if an existing materialized view needs to be replaced. */ + if (is_matview) + { + LOCKMODE lockmode; - /* - * Create the relation. (This will error out if there's an existing view, - * so we don't need more code to complain if "replace" is false.) - */ - intoRelationAddr = DefineRelation(create, relkind, InvalidOid, NULL, NULL); + lockmode = into->replace ? AccessExclusiveLock : NoLock; + (void) RangeVarGetAndCheckCreationNamespace(into->rel, lockmode, + &matviewOid); + replace = OidIsValid(matviewOid) && into->replace; + } - /* - * If necessary, create a TOAST table for the target table. Note that - * NewRelationCreateToastTable ends with CommandCounterIncrement(), so - * that the TOAST table will be visible for insertion. - */ - CommandCounterIncrement(); + if (is_matview && replace) + { + Relation rel; + List *atcmds = NIL; + AlterTableCmd *atcmd; + TupleDesc descriptor; + + rel = relation_open(matviewOid, NoLock); + + if (rel->rd_rel->relkind != RELKIND_MATVIEW) + ereport(ERROR, + errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not a materialized view", + RelationGetRelationName(rel))); + + CheckTableNotInUse(rel, "CREATE OR REPLACE MATERIALIZED VIEW"); + + descriptor = BuildDescForRelation(attrList); + checkViewColumns(descriptor, rel->rd_att, true); + + /* Add new attributes via ALTER TABLE. */ + if (list_length(attrList) > rel->rd_att->natts) + { + ListCell *c; + int skip = rel->rd_att->natts; + + foreach(c, attrList) + { + if (skip > 0) + { + skip--; + continue; + } + atcmd = makeNode(AlterTableCmd); + atcmd->subtype = AT_AddColumnToView; + atcmd->def = (Node *) lfirst(c); + atcmds = lappend(atcmds, atcmd); + } + } + + /* Set access method via ALTER TABLE. */ + if (into->accessMethod != NULL) + { + atcmd = makeNode(AlterTableCmd); + atcmd->subtype = AT_SetAccessMethod; + atcmd->name = into->accessMethod; + atcmds = lappend(atcmds, atcmd); + } + + /* Set tablespace via ALTER TABLE. */ + if (into->tableSpaceName != NULL) + { + atcmd = makeNode(AlterTableCmd); + atcmd->subtype = AT_SetTableSpace; + atcmd->name = into->tableSpaceName; + atcmds = lappend(atcmds, atcmd); + } + + /* Set storage parameters via ALTER TABLE. */ + if (into->options != NIL) + { + atcmd = makeNode(AlterTableCmd); + atcmd->subtype = AT_ReplaceRelOptions; + atcmd->def = (Node *) into->options; + atcmds = lappend(atcmds, atcmd); + } + + if (atcmds != NIL) + { + AlterTableInternal(matviewOid, atcmds, true); + CommandCounterIncrement(); + } + + relation_close(rel, NoLock); + + ObjectAddressSet(intoRelationAddr, RelationRelationId, matviewOid); + } + else + { + CreateStmt *create = makeNode(CreateStmt); + Datum toast_options; + static char *validnsps[] = HEAP_RELOPT_NAMESPACES; + + /* + * Create the target relation by faking up a CREATE TABLE parsetree + * and passing it to DefineRelation. + */ + create->relation = into->rel; + create->tableElts = attrList; + create->inhRelations = NIL; + create->ofTypename = NULL; + create->constraints = NIL; + create->options = into->options; + create->oncommit = into->onCommit; + create->tablespacename = into->tableSpaceName; + create->if_not_exists = false; + create->accessMethod = into->accessMethod; + + /* + * Create the relation. (This will error out if there's an existing + * view, so we don't need more code to complain if "replace" is + * false.) + */ + intoRelationAddr = DefineRelation(create, relkind, InvalidOid, NULL, + NULL); - /* parse and validate reloptions for the toast table */ - toast_options = transformRelOptions((Datum) 0, - create->options, - "toast", - validnsps, - true, false); + /* + * If necessary, create a TOAST table for the target table. Note that + * NewRelationCreateToastTable ends with CommandCounterIncrement(), so + * that the TOAST table will be visible for insertion. + */ + CommandCounterIncrement(); + + /* parse and validate reloptions for the toast table */ + toast_options = transformRelOptions((Datum) 0, + create->options, + "toast", + validnsps, + true, false); - (void) heap_reloptions(RELKIND_TOASTVALUE, toast_options, true); + (void) heap_reloptions(RELKIND_TOASTVALUE, toast_options, true); - NewRelationCreateToastTable(intoRelationAddr.objectId, toast_options); + NewRelationCreateToastTable(intoRelationAddr.objectId, toast_options); + } /* Create the "view" part of a materialized view. */ if (is_matview) @@ -135,7 +231,7 @@ create_ctas_internal(List *attrList, IntoClause *into) /* StoreViewQuery scribbles on tree, so make a copy */ Query *query = (Query *) copyObject(into->viewQuery); - StoreViewQuery(intoRelationAddr.objectId, query, false); + StoreViewQuery(intoRelationAddr.objectId, query, replace); CommandCounterIncrement(); } @@ -236,7 +332,26 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt, /* Check if the relation exists or not */ if (CreateTableAsRelExists(stmt)) + { + /* An existing materialized view can be replaced. */ + if (is_matview && into->replace) + { + RefreshMatViewStmt *refresh; + + /* Change the relation to match the new query and other options. */ + (void) create_ctas_nodata(query->targetList, into); + + /* Refresh the materialized view with a fake statement. */ + refresh = makeNode(RefreshMatViewStmt); + refresh->relation = into->rel; + refresh->skipData = into->skipData; + refresh->concurrent = false; + + return ExecRefreshMatView(refresh, NULL, NULL, NULL); + } + return InvalidObjectAddress; + } /* * Create the tuple receiver object and insert info it will need @@ -397,14 +512,15 @@ CreateTableAsRelExists(CreateTableAsStmt *ctas) oldrelid = get_relname_relid(into->rel->relname, nspid); if (OidIsValid(oldrelid)) { - if (!ctas->if_not_exists) + if (!ctas->if_not_exists && !into->replace) ereport(ERROR, (errcode(ERRCODE_DUPLICATE_TABLE), errmsg("relation \"%s\" already exists", into->rel->relname))); /* - * The relation exists and IF NOT EXISTS has been specified. + * The relation exists and IF NOT EXISTS or OR REPLACE has been + * specified. * * If we are in an extension script, insist that the pre-existing * object be a member of the extension, to avoid security risks. @@ -412,11 +528,12 @@ CreateTableAsRelExists(CreateTableAsStmt *ctas) ObjectAddressSet(address, RelationRelationId, oldrelid); checkMembershipInCurrentExtension(&address); - /* OK to skip */ - ereport(NOTICE, - (errcode(ERRCODE_DUPLICATE_TABLE), - errmsg("relation \"%s\" already exists, skipping", - into->rel->relname))); + if (ctas->if_not_exists) + /* OK to skip */ + ereport(NOTICE, + (errcode(ERRCODE_DUPLICATE_TABLE), + errmsg("relation \"%s\" already exists, skipping", + into->rel->relname))); return true; } diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 9e9dc5c2c1..4660aa1230 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -4484,7 +4484,7 @@ AlterTableGetLockLevel(List *cmds) * Subcommands that may be visible to concurrent SELECTs */ case AT_DropColumn: /* change visible to SELECT */ - case AT_AddColumnToView: /* CREATE VIEW */ + case AT_AddColumnToView: /* via CREATE OR REPLACE [MATERIALIZED] VIEW */ case AT_DropOids: /* used to equiv to DropColumn */ case AT_EnableAlwaysRule: /* may change SELECT rules */ case AT_EnableReplicaRule: /* may change SELECT rules */ @@ -4795,8 +4795,8 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, /* Recursion occurs during execution phase */ pass = AT_PASS_ADD_COL; break; - case AT_AddColumnToView: /* add column via CREATE OR REPLACE VIEW */ - ATSimplePermissions(cmd->subtype, rel, ATT_VIEW); + case AT_AddColumnToView: /* via CREATE OR REPLACE [MATERIALIZED] VIEW */ + ATSimplePermissions(cmd->subtype, rel, ATT_VIEW | ATT_MATVIEW); ATPrepAddColumn(wqueue, rel, recurse, recursing, true, cmd, lockmode, context); /* Recursion occurs during execution phase */ @@ -5220,7 +5220,7 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, switch (cmd->subtype) { case AT_AddColumn: /* ADD COLUMN */ - case AT_AddColumnToView: /* add column via CREATE OR REPLACE VIEW */ + case AT_AddColumnToView: /* via CREATE OR REPLACE [MATERIALIZED] VIEW */ address = ATExecAddColumn(wqueue, tab, rel, &cmd, cmd->recurse, false, lockmode, cur_pass, context); diff --git a/src/backend/commands/view.c b/src/backend/commands/view.c index fdad833832..76532aa35d 100644 --- a/src/backend/commands/view.c +++ b/src/backend/commands/view.c @@ -30,8 +30,6 @@ #include "utils/lsyscache.h" #include "utils/rel.h" -static void checkViewColumns(TupleDesc newdesc, TupleDesc olddesc); - /*--------------------------------------------------------------------- * DefineVirtualRelation * @@ -130,7 +128,7 @@ DefineVirtualRelation(RangeVar *relation, List *tlist, bool replace, * column list. */ descriptor = BuildDescForRelation(attrList); - checkViewColumns(descriptor, rel->rd_att); + checkViewColumns(descriptor, rel->rd_att, false); /* * If new attributes have been added, we must add pg_attribute entries @@ -263,15 +261,22 @@ DefineVirtualRelation(RangeVar *relation, List *tlist, bool replace, * added to generate specific complaints. Also, we allow the new view to have * more columns than the old. */ -static void -checkViewColumns(TupleDesc newdesc, TupleDesc olddesc) +void +checkViewColumns(TupleDesc newdesc, TupleDesc olddesc, bool is_matview) { int i; if (newdesc->natts < olddesc->natts) - ereport(ERROR, - (errcode(ERRCODE_INVALID_TABLE_DEFINITION), - errmsg("cannot drop columns from view"))); + { + if (is_matview) + ereport(ERROR, + errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot drop columns from materialized view")); + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot drop columns from view"))); + } for (i = 0; i < olddesc->natts; i++) { @@ -280,17 +285,34 @@ checkViewColumns(TupleDesc newdesc, TupleDesc olddesc) /* XXX msg not right, but we don't support DROP COL on view anyway */ if (newattr->attisdropped != oldattr->attisdropped) - ereport(ERROR, - (errcode(ERRCODE_INVALID_TABLE_DEFINITION), - errmsg("cannot drop columns from view"))); + { + if (is_matview) + ereport(ERROR, + errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot drop columns from materialized view")); + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot drop columns from view"))); + } if (strcmp(NameStr(newattr->attname), NameStr(oldattr->attname)) != 0) - ereport(ERROR, - (errcode(ERRCODE_INVALID_TABLE_DEFINITION), - errmsg("cannot change name of view column \"%s\" to \"%s\"", - NameStr(oldattr->attname), - NameStr(newattr->attname)), - errhint("Use ALTER VIEW ... RENAME COLUMN ... to change name of view column instead."))); + { + if (is_matview) + ereport(ERROR, + errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot change name of materialized view column \"%s\" to \"%s\"", + NameStr(oldattr->attname), + NameStr(newattr->attname)), + errhint("Use ALTER MATERIALIZED VIEW ... RENAME COLUMN ... to change name of materialized view column instead.")); + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot change name of view column \"%s\" to \"%s\"", + NameStr(oldattr->attname), + NameStr(newattr->attname)), + errhint("Use ALTER VIEW ... RENAME COLUMN ... to change name of view column instead."))); + } /* * We cannot allow type, typmod, or collation to change, since these @@ -299,26 +321,48 @@ checkViewColumns(TupleDesc newdesc, TupleDesc olddesc) */ if (newattr->atttypid != oldattr->atttypid || newattr->atttypmod != oldattr->atttypmod) - ereport(ERROR, - (errcode(ERRCODE_INVALID_TABLE_DEFINITION), - errmsg("cannot change data type of view column \"%s\" from %s to %s", - NameStr(oldattr->attname), - format_type_with_typemod(oldattr->atttypid, - oldattr->atttypmod), - format_type_with_typemod(newattr->atttypid, - newattr->atttypmod)))); + { + if (is_matview) + ereport(ERROR, + errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot change data type of materialized view column \"%s\" from %s to %s", + NameStr(oldattr->attname), + format_type_with_typemod(oldattr->atttypid, + oldattr->atttypmod), + format_type_with_typemod(newattr->atttypid, + newattr->atttypmod))); + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot change data type of view column \"%s\" from %s to %s", + NameStr(oldattr->attname), + format_type_with_typemod(oldattr->atttypid, + oldattr->atttypmod), + format_type_with_typemod(newattr->atttypid, + newattr->atttypmod)))); + } /* * At this point, attcollations should be both valid or both invalid, * so applying get_collation_name unconditionally should be fine. */ if (newattr->attcollation != oldattr->attcollation) - ereport(ERROR, - (errcode(ERRCODE_INVALID_TABLE_DEFINITION), - errmsg("cannot change collation of view column \"%s\" from \"%s\" to \"%s\"", - NameStr(oldattr->attname), - get_collation_name(oldattr->attcollation), - get_collation_name(newattr->attcollation)))); + { + if (is_matview) + ereport(ERROR, + errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot change collation of materialized view column \"%s\" from \"%s\" to \"%s\"", + NameStr(oldattr->attname), + get_collation_name(oldattr->attcollation), + get_collation_name(newattr->attcollation))); + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot change collation of view column \"%s\" from \"%s\" to \"%s\"", + NameStr(oldattr->attname), + get_collation_name(oldattr->attcollation), + get_collation_name(newattr->attcollation)))); + } } /* diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index a043fd4c66..ed477806d1 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -4831,6 +4831,21 @@ CreateMatViewStmt: $8->skipData = !($11); $$ = (Node *) ctas; } + | CREATE OR REPLACE OptNoLog MATERIALIZED VIEW create_mv_target AS SelectStmt opt_with_data + { + CreateTableAsStmt *ctas = makeNode(CreateTableAsStmt); + + ctas->query = $9; + ctas->into = $7; + ctas->objtype = OBJECT_MATVIEW; + ctas->is_select_into = false; + ctas->if_not_exists = false; + /* cram additional flags into the IntoClause */ + $7->rel->relpersistence = $4; + $7->skipData = !($10); + $7->replace = true; + $$ = (Node *) ctas; + } ; create_mv_target: diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index d453e224d9..f4d7f99bc5 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1811,7 +1811,7 @@ psql_completion(const char *text, int start, int end) /* complete with something you can create or replace */ else if (TailMatches("CREATE", "OR", "REPLACE")) COMPLETE_WITH("FUNCTION", "PROCEDURE", "LANGUAGE", "RULE", "VIEW", - "AGGREGATE", "TRANSFORM", "TRIGGER"); + "AGGREGATE", "TRANSFORM", "TRIGGER", "MATERIALIZED VIEW"); /* DROP, but not DROP embedded in other commands */ /* complete with something you can drop */ diff --git a/src/include/commands/view.h b/src/include/commands/view.h index d2d8588989..7eacdaaceb 100644 --- a/src/include/commands/view.h +++ b/src/include/commands/view.h @@ -22,4 +22,7 @@ extern ObjectAddress DefineView(ViewStmt *stmt, const char *queryString, extern void StoreViewQuery(Oid viewOid, Query *viewParse, bool replace); +extern void checkViewColumns(TupleDesc newdesc, TupleDesc olddesc, + bool is_matview); + #endif /* VIEW_H */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 85a62b538e..cbc4b608e3 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2351,7 +2351,7 @@ typedef struct AlterTableStmt typedef enum AlterTableType { AT_AddColumn, /* add column */ - AT_AddColumnToView, /* implicitly via CREATE OR REPLACE VIEW */ + AT_AddColumnToView, /* implicitly via CREATE OR REPLACE [MATERIALIZED] VIEW */ AT_ColumnDefault, /* alter column default */ AT_CookedColumnDefault, /* add a pre-cooked column default */ AT_DropNotNull, /* alter column drop not null */ diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index ea47652adb..4b0ee5d10d 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -168,6 +168,7 @@ typedef struct IntoClause /* materialized view's SELECT query */ Node *viewQuery pg_node_attr(query_jumble_ignore); bool skipData; /* true for WITH NO DATA */ + bool replace; /* replace existing matview? */ } IntoClause; diff --git a/src/test/regress/expected/matview.out b/src/test/regress/expected/matview.out index 038ab73517..e2e2a13396 100644 --- a/src/test/regress/expected/matview.out +++ b/src/test/regress/expected/matview.out @@ -694,3 +694,194 @@ NOTICE: relation "matview_ine_tab" already exists, skipping (0 rows) DROP MATERIALIZED VIEW matview_ine_tab; +-- +-- test CREATE OR REPLACE MATERIALIZED VIEW +-- +-- matview does not already exist +DROP MATERIALIZED VIEW IF EXISTS mvtest_replace; +NOTICE: materialized view "mvtest_replace" does not exist, skipping +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 1 AS a; +SELECT * FROM mvtest_replace; + a +--- + 1 +(1 row) + +-- replace query with data +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 2 AS a; +SELECT * FROM mvtest_replace; + a +--- + 2 +(1 row) + +-- replace query without data +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 3 AS a + WITH NO DATA; +SELECT * FROM mvtest_replace; -- error: not populated +ERROR: materialized view "mvtest_replace" has not been populated +HINT: Use the REFRESH MATERIALIZED VIEW command. +REFRESH MATERIALIZED VIEW mvtest_replace; +SELECT * FROM mvtest_replace; + a +--- + 3 +(1 row) + +-- add column +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 4 AS a, 1 b; +SELECT * FROM mvtest_replace; + a | b +---+--- + 4 | 1 +(1 row) + +-- replace table options +SELECT m.*, c.relname, c.reloptions, s.spcname, a.amname + FROM mvtest_replace m + CROSS JOIN pg_class c + LEFT JOIN pg_tablespace s ON s.oid = c.reltablespace + LEFT JOIN pg_am a ON a.oid = c.relam + WHERE c.relname = 'mvtest_replace'; + a | b | relname | reloptions | spcname | amname +---+---+----------------+------------+---------+-------- + 4 | 1 | mvtest_replace | | | heap +(1 row) + +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace + USING heap2 + WITH (fillfactor = 50) + TABLESPACE regress_tblspace + AS SELECT 5 AS a, 1 AS b; +SELECT m.*, c.relname, c.reloptions, s.spcname, a.amname + FROM mvtest_replace m + CROSS JOIN pg_class c + LEFT JOIN pg_tablespace s ON s.oid = c.reltablespace + LEFT JOIN pg_am a ON a.oid = c.relam + WHERE c.relname = 'mvtest_replace'; + a | b | relname | reloptions | spcname | amname +---+---+----------------+-----------------+------------------+-------- + 5 | 1 | mvtest_replace | {fillfactor=50} | regress_tblspace | heap2 +(1 row) + +-- can replace matview that has a dependent view +CREATE VIEW mvtest_replace_v AS + SELECT * FROM mvtest_replace; +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 6 AS a, 1 AS b; +SELECT * FROM mvtest_replace, mvtest_replace_v; + a | b | a | b +---+---+---+--- + 6 | 1 | 6 | 1 +(1 row) + +DROP VIEW mvtest_replace_v; +-- index gets rebuilt when replacing with data +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 7 AS a, 1 AS b; +CREATE UNIQUE INDEX ON mvtest_replace (b); +SELECT * FROM mvtest_replace; + a | b +---+--- + 7 | 1 +(1 row) + +SET enable_seqscan = off; -- force index scan +EXPLAIN (COSTS OFF) SELECT * FROM mvtest_replace WHERE b = 1; + QUERY PLAN +--------------------------------------------------------- + Index Scan using mvtest_replace_b_idx on mvtest_replace + Index Cond: (b = 1) +(2 rows) + +SELECT * FROM mvtest_replace WHERE b = 1; + a | b +---+--- + 7 | 1 +(1 row) + +RESET enable_seqscan; +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 8 AS a, 1 AS b; +SET enable_seqscan = off; -- force index scan +EXPLAIN (COSTS OFF) SELECT * FROM mvtest_replace WHERE b = 1; + QUERY PLAN +--------------------------------------------------------- + Index Scan using mvtest_replace_b_idx on mvtest_replace + Index Cond: (b = 1) +(2 rows) + +SELECT * FROM mvtest_replace WHERE b = 1; + a | b +---+--- + 8 | 1 +(1 row) + +RESET enable_seqscan; +-- cannot change column data type +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 9 AS a, 'x' AS b; -- error +ERROR: cannot change data type of materialized view column "b" from integer to text +SELECT * FROM mvtest_replace; + a | b +---+--- + 8 | 1 +(1 row) + +-- cannot rename column +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 10 AS a, 1 AS b2; -- error +ERROR: cannot change name of materialized view column "b" to "b2" +HINT: Use ALTER MATERIALIZED VIEW ... RENAME COLUMN ... to change name of materialized view column instead. +SELECT * FROM mvtest_replace; + a | b +---+--- + 8 | 1 +(1 row) + +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 11 AS a, 1 AS b, 'y' COLLATE "C" AS c; +SELECT * FROM mvtest_replace; + a | b | c +----+---+--- + 11 | 1 | y +(1 row) + +-- cannot change column collation +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 12 AS a, 1 AS b, 'x' COLLATE "POSIX" AS c; -- error +ERROR: cannot change collation of materialized view column "c" from "C" to "POSIX" +SELECT * FROM mvtest_replace; + a | b | c +----+---+--- + 11 | 1 | y +(1 row) + +-- cannot drop column +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 13 AS a, 1 AS b; -- error +ERROR: cannot drop columns from materialized view +SELECT * FROM mvtest_replace; + a | b | c +----+---+--- + 11 | 1 | y +(1 row) + +-- must target a matview +CREATE VIEW mvtest_not_mv AS + SELECT 1 AS a; +CREATE OR REPLACE MATERIALIZED VIEW mvtest_not_mv AS + SELECT 1 AS a; -- error +ERROR: "mvtest_not_mv" is not a materialized view +DROP VIEW mvtest_not_mv; +-- cannot use OR REPLACE with IF NOT EXISTS +CREATE OR REPLACE MATERIALIZED VIEW IF NOT EXISTS mvtest_replace AS + SELECT 1 AS a; +ERROR: syntax error at or near "NOT" +LINE 1: CREATE OR REPLACE MATERIALIZED VIEW IF NOT EXISTS mvtest_rep... + ^ +DROP MATERIALIZED VIEW mvtest_replace; diff --git a/src/test/regress/sql/matview.sql b/src/test/regress/sql/matview.sql index b74ee305e0..c12f0243c9 100644 --- a/src/test/regress/sql/matview.sql +++ b/src/test/regress/sql/matview.sql @@ -314,3 +314,111 @@ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) CREATE MATERIALIZED VIEW IF NOT EXISTS matview_ine_tab AS SELECT 1 / 0 WITH NO DATA; -- ok DROP MATERIALIZED VIEW matview_ine_tab; + +-- +-- test CREATE OR REPLACE MATERIALIZED VIEW +-- + +-- matview does not already exist +DROP MATERIALIZED VIEW IF EXISTS mvtest_replace; +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 1 AS a; +SELECT * FROM mvtest_replace; + +-- replace query with data +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 2 AS a; +SELECT * FROM mvtest_replace; + +-- replace query without data +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 3 AS a + WITH NO DATA; +SELECT * FROM mvtest_replace; -- error: not populated +REFRESH MATERIALIZED VIEW mvtest_replace; +SELECT * FROM mvtest_replace; + +-- add column +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 4 AS a, 1 b; +SELECT * FROM mvtest_replace; + +-- replace table options +SELECT m.*, c.relname, c.reloptions, s.spcname, a.amname + FROM mvtest_replace m + CROSS JOIN pg_class c + LEFT JOIN pg_tablespace s ON s.oid = c.reltablespace + LEFT JOIN pg_am a ON a.oid = c.relam + WHERE c.relname = 'mvtest_replace'; +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace + USING heap2 + WITH (fillfactor = 50) + TABLESPACE regress_tblspace + AS SELECT 5 AS a, 1 AS b; +SELECT m.*, c.relname, c.reloptions, s.spcname, a.amname + FROM mvtest_replace m + CROSS JOIN pg_class c + LEFT JOIN pg_tablespace s ON s.oid = c.reltablespace + LEFT JOIN pg_am a ON a.oid = c.relam + WHERE c.relname = 'mvtest_replace'; + +-- can replace matview that has a dependent view +CREATE VIEW mvtest_replace_v AS + SELECT * FROM mvtest_replace; +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 6 AS a, 1 AS b; +SELECT * FROM mvtest_replace, mvtest_replace_v; +DROP VIEW mvtest_replace_v; + +-- index gets rebuilt when replacing with data +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 7 AS a, 1 AS b; +CREATE UNIQUE INDEX ON mvtest_replace (b); +SELECT * FROM mvtest_replace; +SET enable_seqscan = off; -- force index scan +EXPLAIN (COSTS OFF) SELECT * FROM mvtest_replace WHERE b = 1; +SELECT * FROM mvtest_replace WHERE b = 1; +RESET enable_seqscan; +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 8 AS a, 1 AS b; +SET enable_seqscan = off; -- force index scan +EXPLAIN (COSTS OFF) SELECT * FROM mvtest_replace WHERE b = 1; +SELECT * FROM mvtest_replace WHERE b = 1; +RESET enable_seqscan; + +-- cannot change column data type +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 9 AS a, 'x' AS b; -- error +SELECT * FROM mvtest_replace; + +-- cannot rename column +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 10 AS a, 1 AS b2; -- error +SELECT * FROM mvtest_replace; + +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 11 AS a, 1 AS b, 'y' COLLATE "C" AS c; +SELECT * FROM mvtest_replace; + +-- cannot change column collation +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 12 AS a, 1 AS b, 'x' COLLATE "POSIX" AS c; -- error +SELECT * FROM mvtest_replace; + +-- cannot drop column +CREATE OR REPLACE MATERIALIZED VIEW mvtest_replace AS + SELECT 13 AS a, 1 AS b; -- error +SELECT * FROM mvtest_replace; + +-- must target a matview +CREATE VIEW mvtest_not_mv AS + SELECT 1 AS a; +CREATE OR REPLACE MATERIALIZED VIEW mvtest_not_mv AS + SELECT 1 AS a; -- error +DROP VIEW mvtest_not_mv; + +-- cannot use OR REPLACE with IF NOT EXISTS +CREATE OR REPLACE MATERIALIZED VIEW IF NOT EXISTS mvtest_replace AS + SELECT 1 AS a; + +DROP MATERIALIZED VIEW mvtest_replace; -- 2.45.2 --kl5hblbqqgigaot5 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v1-0002-Deprecate-CREATE-MATERIALIZED-VIEW-IF-NOT-EXISTS.patch" ^ permalink raw reply [nested|flat] 30+ messages in thread
end of thread, other threads:[~2024-05-21 16:35 UTC | newest] Thread overview: 30+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2019-11-20 20:53 [PATCH v3] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-11-20 20:53 [PATCH v1] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-11-20 20:53 [PATCH v2 1/2] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v21 1/2] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v14 2/3] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v29 1/3] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v25 1/4] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v19 1/2] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v22 1/3] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v23 1/3] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v15 1/2] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v8 4/5] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v10 1/5] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v12 1/5] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v13 1/5] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v27 1/5] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v16 1/3] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v9 2/3] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v6 4/6] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v7 4/6] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v5 4/5] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v28 1/5] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v17 1/2] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v18] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v20 1/2] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v24 1/4] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v11 1/6] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v4 4/5] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2019-12-13 02:54 [PATCH v19] vacuum errcontext to show block being processed Justin Pryzby <[email protected]> 2024-05-21 16:35 [PATCH v1 1/2] Add CREATE OR REPLACE MATERIALIZED VIEW Erik Wienhold <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox