agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v4] fix HOT tuples while scanning for index builds 21+ messages / 2 participants [nested] [flat]
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH v4] fix HOT tuples while scanning for index builds @ 2020-08-12 18:02 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Alvaro Herrera @ 2020-08-12 18:02 UTC (permalink / raw) --- src/backend/access/heap/heapam_handler.c | 20 ++++++++++++++++++++ src/backend/access/heap/pruneheap.c | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 267a6ee25a..ba44e30035 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1324,6 +1324,12 @@ heapam_index_build_range_scan(Relation heapRelation, * buffer continuously while visiting the page, so no pruning * operation can occur either. * + * In cases with only ShareUpdateExclusiveLock on the table, it's + * possible for some HOT tuples to appear that we didn't know about + * when we first read the page. To handle that case, we re-obtain the + * list of root offsets when a HOT tuple points to a root item that we + * don't know about. + * * Also, although our opinions about tuple liveness could change while * we scan the page (due to concurrent transaction commits/aborts), * the chain root locations won't, so this info doesn't need to be @@ -1625,6 +1631,20 @@ heapam_index_build_range_scan(Relation heapRelation, offnum = ItemPointerGetOffsetNumber(&heapTuple->t_self); + /* + * If a HOT tuple points to a root that we don't know + * about, obtain root items afresh. If that still fails, + * report it as corruption. + */ + if (root_offsets[offnum - 1] == InvalidOffsetNumber) + { + Page page = BufferGetPage(hscan->rs_cbuf); + + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); + heap_get_root_tuples(page, root_offsets); + LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_UNLOCK); + } + if (!OffsetNumberIsValid(root_offsets[offnum - 1])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 256df4de10..7e3d44dfd6 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -732,7 +732,7 @@ heap_page_prune_execute(Buffer buffer, * root_offsets[k - 1] = j. * * The passed-in root_offsets array must have MaxHeapTuplesPerPage entries. - * We zero out all unused entries. + * Unused entries are filled with InvalidOffsetNumber (zero). * * The function must be called with at least share lock on the buffer, to * prevent concurrent prune operations. @@ -747,7 +747,8 @@ heap_get_root_tuples(Page page, OffsetNumber *root_offsets) OffsetNumber offnum, maxoff; - MemSet(root_offsets, 0, MaxHeapTuplesPerPage * sizeof(OffsetNumber)); + MemSet(root_offsets, InvalidOffsetNumber, + MaxHeapTuplesPerPage * sizeof(OffsetNumber)); maxoff = PageGetMaxOffsetNumber(page); for (offnum = FirstOffsetNumber; offnum <= maxoff; offnum = OffsetNumberNext(offnum)) -- 2.20.1 --opJtzjQTFsWo+cga-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: Showing I/O timings spent reading/writing temp buffers in EXPLAIN @ 2022-03-01 00:46 Masahiko Sawada <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Masahiko Sawada @ 2022-03-01 00:46 UTC (permalink / raw) To: Melanie Plageman <[email protected]>; +Cc: Ranier Vilela <[email protected]>; pgsql-hackers; [email protected]; [email protected] Hi, Sorry for the late reply. On Fri, Nov 19, 2021 at 7:24 AM Melanie Plageman <[email protected]> wrote: > > On Sun, Aug 22, 2021 at 9:47 PM Masahiko Sawada <[email protected]> wrote: > > > > On Thu, Aug 19, 2021 at 10:52 PM Ranier Vilela <[email protected]> wrote: > > > > > > Em qui., 19 de ago. de 2021 às 09:21, Masahiko Sawada <[email protected]> escreveu: > > > > > The presentation seems a little confusing, wouldn't it be better? > > > > > > I/O Timings: shared/local read=xxxx write=xxx temp read=0.487 write=2.073 > > > > Yeah, it looks better to add "shared/local". > > Using the patch, I do feel like the EXPLAIN format of > > shared/local xxx, temp xxx > > is a bit confusing. If temp is going to be its own EXPLAIN IO timing > category (as opposed to being summed with relation data block IO from > local and shared buffers), then it seems like local and shared should > be separated as well. > > shared xxx, local xxx, temp xxx I think the I/O timing shown as shared/local is the time spent on disk I/O so it doesn't matter if these disk I/O are for shared buffers or local buffers. > > With your patch applied, below is the top of the EXPLAIN output for a > query joining a temporary table (so using local buffers) to a regular > table (using shared buffers) and spilling the hash join (temp files). > > Aggregate (actual rows=1 loops=1) > Buffers: shared read=4425, local read=4425 dirtied=4425 > written=4423, temp read=5963 written=5963 > I/O Timings: shared/local read=23.546, temp read=13.309 write=74.198 > > I found that using the same terminology as the "EXPLAIN BUFFERS" output > but not using the same categories was kind of confusing. > > If it is only meaningful to distinguish between relation data IO and > query temp file IO, then maybe the words used in I/O Timings in EXPLAIN > should be "rel data" and "temp" or something like that. But if we do that, we end up using different terminology in "I/O Timing" and "Buffers". I think it's better to use consistent words used in them. Regards, -- Masahiko Sawada EDB: https://www.enterprisedb.com/ ^ permalink raw reply [nested|flat] 21+ messages in thread
end of thread, other threads:[~2022-03-01 00:46 UTC | newest] Thread overview: 21+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2020-08-12 18:02 [PATCH v4] fix HOT tuples while scanning for index builds Alvaro Herrera <[email protected]> 2022-03-01 00:46 Re: Showing I/O timings spent reading/writing temp buffers in EXPLAIN Masahiko Sawada <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox