agora inbox for [email protected]
help / color / mirror / Atom feed[PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
194+ messages / 36 participants
[nested] [flat]
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------B3374ACE3BEE92372E040736
Content-Type: application/sql;
name="test.sql"
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
filename="test.sql"
Y3JlYXRlIGV4dGVuc2lvbiBwZ192aXNpYmlsaXR5OwoKZHJvcCB0YWJsZSBpZiBleGlzdHMg
dDsKY3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwppbnNlcnQgaW50byB0IHNlbGVj
dCBpLCAoc2VsZWN0IHN0cmluZ19hZ2cobWQ1KHJhbmRvbSgpOjp0ZXh0KSwgJycpIGZyb20g
Z2VuZXJhdGVfc2VyaWVzKDEsMTAwKSkgZnJvbSBnZW5lcmF0ZV9zZXJpZXMoMSwxMDAwMDAp
IHMoaSk7CmNvcHkgdCB0byAnL3RtcC90LmRhdGEnOwpkcm9wIHRhYmxlIHQ7CgpiZWdpbjsK
Y3JlYXRlIHRhYmxlIHQgKGEgaW50LCBiIHRleHQpOwpjb3B5IHQgZnJvbSAnL3RtcC90LmRh
dGEnIGZyZWV6ZTsKCnNlbGVjdCAqIGZyb20gcGdfdmlzaWJpbGl0eSgndCcpOwpzZWxlY3Qg
KiBmcm9tIHBnX3Zpc2liaWxpdHkoKHNlbGVjdCByZWx0b2FzdHJlbGlkIGZyb20gcGdfY2xh
c3Mgd2hlcmUgcmVsbmFtZSA9ICd0JykpOwoKCnNlbGVjdCBjb3VudCgqKSBmcm9tIHBnX3Zp
c2liaWxpdHkoJ3QnKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgndCcp
IHdoZXJlIG5vdCBhbGxfdmlzaWJsZTsKCgpzZWxlY3QgY291bnQoKikgZnJvbSBwZ192aXNp
YmlsaXR5KChzZWxlY3QgcmVsdG9hc3RyZWxpZCBmcm9tIHBnX2NsYXNzIHdoZXJlIHJlbG5h
bWUgPSAndCcpKTsKc2VsZWN0IGNvdW50KCopIGZyb20gcGdfdmlzaWJpbGl0eSgoc2VsZWN0
IHJlbHRvYXN0cmVsaWQgZnJvbSBwZ19jbGFzcyB3aGVyZSByZWxuYW1lID0gJ3QnKSkgd2hl
cmUgbm90IGFsbF92aXNpYmxlOwo=
--------------B3374ACE3BEE92372E040736--
^ permalink raw reply [nested|flat] 194+ messages in thread
* [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE
@ 2021-01-10 19:30 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Tomas Vondra @ 2021-01-10 19:30 UTC (permalink / raw)
Make sure COPY FREEZE marks the pages as PD_ALL_VISIBLE and updates the
visibility map. Until now it only marked individual tuples as frozen,
but page-level flags were not updated.
This is a fairly old patch, and multiple people worked on it. The first
version was written by Jeff Janes, and then reworked by Pavan Deolasee
and Anastasia Lubennikova.
Author: Pavan Deolasee, Anastasia Lubennikova, Jeff Janes
Reviewed-by: Kuntal Ghosh, Jeff Janes, Tomas Vondra, Masahiko Sawada, Andres Freund, Ibrar Ahmed, Robert Haas, Tatsuro Ishii
Discussion: https://postgr.es/m/CABOikdN-ptGv0mZntrK2Q8OtfUuAjqaYMGmkdU1dCKFtUxVLrg@mail.gmail.com
Discussion: https://postgr.es/m/CAMkU%3D1w3osJJ2FneELhhNRLxfZitDgp9FPHee08NT2FQFmz_pQ%40mail.gmail.com
---
.../pg_visibility/expected/pg_visibility.out | 64 +++++++++++++++
contrib/pg_visibility/sql/pg_visibility.sql | 77 +++++++++++++++++++
src/backend/access/heap/heapam.c | 76 ++++++++++++++++--
src/backend/access/heap/hio.c | 17 ++++
src/include/access/heapam_xlog.h | 3 +
5 files changed, 229 insertions(+), 8 deletions(-)
diff --git a/contrib/pg_visibility/expected/pg_visibility.out b/contrib/pg_visibility/expected/pg_visibility.out
index ca4b6e186b..0017e3415c 100644
--- a/contrib/pg_visibility/expected/pg_visibility.out
+++ b/contrib/pg_visibility/expected/pg_visibility.out
@@ -179,6 +179,69 @@ select pg_truncate_visibility_map('test_partition');
(1 row)
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | t | t
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | f | f
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+copy copyfreeze from stdin;
+copy copyfreeze from stdin freeze;
+commit;
+select * from pg_visibility_map('copyfreeze');
+ blkno | all_visible | all_frozen
+-------+-------------+------------
+ 0 | t | t
+ 1 | f | f
+ 2 | t | t
+(3 rows)
+
+select * from pg_check_frozen('copyfreeze');
+ t_ctid
+--------
+(0 rows)
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -188,3 +251,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/contrib/pg_visibility/sql/pg_visibility.sql b/contrib/pg_visibility/sql/pg_visibility.sql
index f79b54480b..ec1afd4906 100644
--- a/contrib/pg_visibility/sql/pg_visibility.sql
+++ b/contrib/pg_visibility/sql/pg_visibility.sql
@@ -94,6 +94,82 @@ select count(*) > 0 from pg_visibility_map_summary('test_partition');
select * from pg_check_frozen('test_partition'); -- hopefully none
select pg_truncate_visibility_map('test_partition');
+-- test copy freeze
+create table copyfreeze (a int, b char(1500));
+
+-- load all rows via COPY FREEZE and ensure that all pages are set all-visible
+-- and all-frozen.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- load half the rows via regular COPY and rest via COPY FREEZE. The pages
+-- which are touched by regular COPY must not be set all-visible/all-frozen. On
+-- the other hand, pages allocated by COPY FREEZE should be marked
+-- all-frozen/all-visible.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
+-- Try a mix of regular COPY and COPY FREEZE.
+begin;
+truncate copyfreeze;
+copy copyfreeze from stdin freeze;
+1 '1'
+2 '2'
+3 '3'
+4 '4'
+5 '5'
+\.
+copy copyfreeze from stdin;
+6 '6'
+\.
+copy copyfreeze from stdin freeze;
+7 '7'
+8 '8'
+9 '9'
+10 '10'
+11 '11'
+12 '12'
+\.
+commit;
+select * from pg_visibility_map('copyfreeze');
+select * from pg_check_frozen('copyfreeze');
+
-- cleanup
drop table test_partitioned;
drop view test_view;
@@ -103,3 +179,4 @@ drop server dummy_server;
drop foreign data wrapper dummy;
drop materialized view matview_visibility_test;
drop table regular_table;
+drop table copyfreeze;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 53e997cd55..32cc010cb7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2103,6 +2103,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
int ndone;
PGAlignedBlock scratch;
Page page;
+ Buffer vmbuffer = InvalidBuffer;
bool needwal;
Size saveFreeSpace;
bool need_tuple_data = RelationIsLogicallyLogged(relation);
@@ -2157,8 +2158,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2166,12 +2168,20 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/*
* Find buffer where at least the next tuple will fit. If the page is
* all-visible, this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
page = BufferGetPage(buffer);
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
+ all_frozen_set = true;
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2205,7 +2215,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
log_heap_new_cid(relation, heaptup);
}
- if (PageIsAllVisible(page))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(page);
@@ -2213,6 +2230,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ? See heap_insert()
@@ -2236,8 +2255,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If the page was previously empty, we can reinit the page
* instead of restoring the whole thing.
*/
- init = (ItemPointerGetOffsetNumber(&(heaptuples[ndone]->t_self)) == FirstOffsetNumber &&
- PageGetMaxOffsetNumber(page) == FirstOffsetNumber + nthispage - 1);
+ init = starting_with_empty_page;
/* allocate xl_heap_multi_insert struct from the scratch area */
xlrec = (xl_heap_multi_insert *) scratchptr;
@@ -2255,7 +2273,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
/* the rest of the scratch space is used for tuple data */
tupledata = scratchptr;
- xlrec->flags = all_visible_cleared ? XLH_INSERT_ALL_VISIBLE_CLEARED : 0;
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!(all_visible_cleared && all_frozen_set));
+
+ xlrec->flags = 0;
+ if (all_visible_cleared)
+ xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
+
xlrec->ntuples = nthispage;
/*
@@ -2329,13 +2355,39 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
- ReleaseBuffer(vmbuffer);
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ */
+ if (all_frozen_set)
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
+ UnlockReleaseBuffer(buffer);
ndone += nthispage;
+
+ /*
+ * NB: Only release vmbuffer after inserting all tuples - it's fairly
+ * likely that we'll insert into subsequent heap pages that are likely
+ * to use the same vm page.
+ */
}
+ if (vmbuffer != InvalidBuffer)
+ ReleaseBuffer(vmbuffer);
+
/*
* We're done with the actual inserts. Check for conflicts again, to
* ensure that all rw-conflicts in to these inserts are detected. Without
@@ -8265,6 +8317,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8354,6 +8410,10 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c
index fac3b8e9ff..2d23b3ef71 100644
--- a/src/backend/access/heap/hio.c
+++ b/src/backend/access/heap/hio.c
@@ -433,6 +433,14 @@ loop:
buffer = ReadBufferBI(relation, targetBlock, RBM_NORMAL, bistate);
if (PageIsAllVisible(BufferGetPage(buffer)))
visibilitymap_pin(relation, targetBlock, vmbuffer);
+
+ /*
+ * If the page is empty, pin vmbuffer to set all_frozen bit later.
+ */
+ if ((options & HEAP_INSERT_FROZEN) &&
+ (PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0))
+ visibilitymap_pin(relation, targetBlock, vmbuffer);
+
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
}
else if (otherBlock == targetBlock)
@@ -619,6 +627,15 @@ loop:
PageInit(page, BufferGetPageSize(buffer), 0);
MarkBufferDirty(buffer);
+ /*
+ * The page is empty, pin vmbuffer to set all_frozen bit.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ Assert(PageGetMaxOffsetNumber(BufferGetPage(buffer)) == 0);
+ visibilitymap_pin(relation, BufferGetBlockNumber(buffer), vmbuffer);
+ }
+
/*
* Release the file-extension lock; it's now OK for someone else to extend
* the relation some more.
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 51586b883d..178d49710a 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -69,6 +69,9 @@
#define XLH_INSERT_CONTAINS_NEW_TUPLE (1<<3)
#define XLH_INSERT_ON_TOAST_RELATION (1<<4)
+/* all_frozen_set always implies all_visible_set */
+#define XLH_INSERT_ALL_FROZEN_SET (1<<5)
+
/*
* xl_heap_update flag values, 8 bits are available.
*/
--
2.26.2
--------------19F6AF79E033375862CBEBB4
Content-Type: text/x-patch; charset=UTF-8;
name="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002_handle_HEAP_INSERT_FROZEN_in_heap_insert.patch"
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 32cc010cb7..3663ff4b83 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -1862,8 +1862,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool starting_with_empty_page;
bool all_visible_cleared = false;
+ bool all_frozen_set = false;
+ uint8 vmstatus = 0;
/*
* Fill in tuple header fields and toast the tuple if necessary.
@@ -1876,11 +1880,36 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/*
* Find buffer to insert this tuple into. If the page is all visible,
* this will also pin the requisite visibility map page.
+ *
+ * Also pin visibility map page if COPY FREEZE inserts tuples into an
+ * empty page. See all_frozen_set below.
*/
buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
InvalidBuffer, options, bistate,
&vmbuffer, NULL);
+
+ /*
+ * If we're inserting frozen entry into an empty page,
+ * set visibility map bits and PageAllVisible() hint.
+ *
+ * If we're inserting frozen entry into already all_frozen page,
+ * preserve this state.
+ */
+ if (options & HEAP_INSERT_FROZEN)
+ {
+ page = BufferGetPage(buffer);
+
+ starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
+
+ if (visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer))
+ vmstatus = visibilitymap_get_status(relation,
+ BufferGetBlockNumber(buffer), &vmbuffer);
+
+ if ((starting_with_empty_page || vmstatus & VISIBILITYMAP_ALL_FROZEN))
+ all_frozen_set = true;
+ }
+
/*
* We're about to do the actual insert -- but check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -1904,7 +1933,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * If the page is all visible, need to clear that, unless we're only
+ * going to add further frozen rows to it.
+ *
+ * If we're only adding already frozen rows to a previously empty
+ * or all visible page, mark it as all-visible.
+ */
+ if (PageIsAllVisible(BufferGetPage(buffer)) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
PageClearAllVisible(BufferGetPage(buffer));
@@ -1912,6 +1948,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
+ else if (all_frozen_set)
+ PageSetAllVisible(page);
/*
* XXX Should we set PageSetPrunable on this page ?
@@ -1959,6 +1997,8 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.flags = 0;
if (all_visible_cleared)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
+ if (all_frozen_set)
+ xlrec.flags = XLH_INSERT_ALL_FROZEN_SET;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
@@ -2007,6 +2047,29 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
END_CRIT_SECTION();
+ /*
+ * If we've frozen everything on the page, update the visibilitymap.
+ * We're already holding pin on the vmbuffer.
+ *
+ * No need to update the visibilitymap if it had all_frozen bit set
+ * before this insertion.
+ */
+ if (all_frozen_set && ((vmstatus & VISIBILITYMAP_ALL_FROZEN) == 0))
+ {
+ Assert(PageIsAllVisible(page));
+ Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
+
+ /*
+ * It's fine to use InvalidTransactionId here - this is only used
+ * when HEAP_INSERT_FROZEN is specified, which intentionally
+ * violates visibility rules.
+ */
+ visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
+ InvalidXLogRecPtr, vmbuffer,
+ InvalidTransactionId,
+ VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
+ }
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -8197,6 +8260,10 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
+ /* check that the mutually exclusive flags are not both set */
+ Assert (!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
+ (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
+
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
@@ -8267,6 +8334,11 @@ heap_xlog_insert(XLogReaderState *record)
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
+ /* XLH_INSERT_ALL_FROZEN_SET implies that all tuples are visible */
+ if (xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)
+ PageSetAllVisible(page);
+
+
MarkBufferDirty(buffer);
}
if (BufferIsValid(buffer))
--------------19F6AF79E033375862CBEBB4--
^ permalink raw reply [nested|flat] 194+ messages in thread
* First draft of PG 17 release notes
@ 2024-05-09 04:03 Bruce Momjian <[email protected]>
2024-05-09 04:44 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-09 04:53 ` Re: First draft of PG 17 release notes Bertrand Drouvot <[email protected]>
2024-05-09 05:17 ` Re: First draft of PG 17 release notes Masahiko Sawada <[email protected]>
2024-05-09 06:37 ` Re: First draft of PG 17 release notes Richard Guo <[email protected]>
2024-05-09 09:18 ` Re: First draft of PG 17 release notes Aleksander Alekseev <[email protected]>
2024-05-09 10:00 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 10:22 ` Re: First draft of PG 17 release notes Dagfinn Ilmari Mannsåker <[email protected]>
2024-05-09 10:53 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 16:10 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
2024-05-09 20:05 ` Re: First draft of PG 17 release notes Thomas Munro <[email protected]>
2024-05-10 08:24 ` Re: First draft of PG 17 release notes Bharath Rupireddy <[email protected]>
2024-05-10 16:29 ` Re: First draft of PG 17 release notes Daniel Verite <[email protected]>
2024-05-10 16:50 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-11 05:27 ` Re: First draft of PG 17 release notes Andy Fan <[email protected]>
2024-05-11 19:32 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
2024-05-14 10:34 ` Re: First draft of PG 17 release notes Elena Indrupskaya <[email protected]>
2024-05-14 13:58 ` Re: First draft of PG 17 release notes Pantelis Theodosiou <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-16 02:39 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-16 08:29 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-17 13:22 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-17 13:42 ` Re: First draft of PG 17 release notes Daniel Verite <[email protected]>
2024-05-17 20:30 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-21 02:20 ` Re: First draft of PG 17 release notes Amit Langote <[email protected]>
2024-05-22 02:29 ` Re: First draft of PG 17 release notes Masahiko Sawada <[email protected]>
2024-05-22 12:25 ` Re: First draft of PG 17 release notes torikoshia <[email protected]>
2024-05-23 11:22 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-06-05 22:46 ` Re: First draft of PG 17 release notes Dean Rasheed <[email protected]>
0 siblings, 28 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 04:03 UTC (permalink / raw)
To: pgsql-hackers
I have committed the first draft of the PG 17 release notes; you can
see the results here:
https://momjian.us/pgsql_docs/release-17.html
It will be improved until the final release. The item count is 188,
which is similar to recent releases:
release-10: 189
release-11: 170
release-12: 180
release-13: 178
release-14: 220
release-15: 184
release-16: 206
release-17: 188
I welcome feedback. For some reason it was an easier job than usual.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-09 04:44 ` David Rowley <[email protected]>
2024-05-09 04:47 ` Re: First draft of PG 17 release notes Muhammad Ikram <[email protected]>
2024-05-09 13:08 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 2 replies; 194+ messages in thread
From: David Rowley @ 2024-05-09 04:44 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, 9 May 2024 at 16:04, Bruce Momjian <[email protected]> wrote:
> I welcome feedback. For some reason it was an easier job than usual.
Thanks for working on that.
> +2023-11-02 [cac169d68] Increase DEFAULT_FDW_TUPLE_COST from 0.01 to 0.2
> +Double the default foreign data wrapper tuple cost (David Rowley, Umair Shahid)
That's 20x rather than 2x.
David
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 04:44 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
@ 2024-05-09 04:47 ` Muhammad Ikram <[email protected]>
2024-05-09 04:52 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-09 13:10 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
1 sibling, 2 replies; 194+ messages in thread
From: Muhammad Ikram @ 2024-05-09 04:47 UTC (permalink / raw)
To: David Rowley <[email protected]>; +Cc: Bruce Momjian <[email protected]>; pgsql-hackers
Hi Bruce,
A minor formatting issue in the start below. Bullet is not required here.
E.1.1. Overview
<https://momjian.us/pgsql_docs/release-17.html#RELEASE-17-HIGHLIGHTS;
PostgreSQL 17 contains many new features and enhancements, including:
-
The above items and other new features of PostgreSQL 17 are explained in
more detail in the sections below.
Regards,
Ikram
On Thu, May 9, 2024 at 9:45 AM David Rowley <[email protected]> wrote:
> On Thu, 9 May 2024 at 16:04, Bruce Momjian <[email protected]> wrote:
> > I welcome feedback. For some reason it was an easier job than usual.
>
> Thanks for working on that.
>
> > +2023-11-02 [cac169d68] Increase DEFAULT_FDW_TUPLE_COST from 0.01 to 0.2
>
> > +Double the default foreign data wrapper tuple cost (David Rowley, Umair
> Shahid)
>
> That's 20x rather than 2x.
>
> David
>
>
>
--
Muhammad Ikram
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 04:44 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-09 04:47 ` Re: First draft of PG 17 release notes Muhammad Ikram <[email protected]>
@ 2024-05-09 04:52 ` David Rowley <[email protected]>
2024-05-09 13:11 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
1 sibling, 1 reply; 194+ messages in thread
From: David Rowley @ 2024-05-09 04:52 UTC (permalink / raw)
To: Muhammad Ikram <[email protected]>; +Cc: Bruce Momjian <[email protected]>; pgsql-hackers
On Thu, 9 May 2024 at 16:47, Muhammad Ikram <[email protected]> wrote:
> A minor formatting issue in the start below. Bullet is not required here.
This is a placeholder for the highlight features of v17 will go.
Bruce tends not to decide what those are all by himself.
David
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 04:44 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-09 04:47 ` Re: First draft of PG 17 release notes Muhammad Ikram <[email protected]>
2024-05-09 04:52 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
@ 2024-05-09 13:11 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 13:11 UTC (permalink / raw)
To: David Rowley <[email protected]>; +Cc: Muhammad Ikram <[email protected]>; pgsql-hackers
On Thu, May 9, 2024 at 04:52:14PM +1200, David Rowley wrote:
> On Thu, 9 May 2024 at 16:47, Muhammad Ikram <[email protected]> wrote:
> > A minor formatting issue in the start below. Bullet is not required here.
>
> This is a placeholder for the highlight features of v17 will go.
> Bruce tends not to decide what those are all by himself.
Yes, I already have so much of my opinion in the release notes that I
prefer others to make that list, and to make the Acknowledgments list
at the bottom.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 04:44 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-09 04:47 ` Re: First draft of PG 17 release notes Muhammad Ikram <[email protected]>
@ 2024-05-09 13:10 ` Bruce Momjian <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 13:10 UTC (permalink / raw)
To: Muhammad Ikram <[email protected]>; +Cc: David Rowley <[email protected]>; pgsql-hackers
On Thu, May 9, 2024 at 09:47:34AM +0500, Muhammad Ikram wrote:
> Hi Bruce,
>
> A minor formatting issue in the start below. Bullet is not required here.
>
>
> E.1.1. Overview
>
> PostgreSQL 17 contains many new features and enhancements, including:
>
> •
>
> The above items and other new features of PostgreSQL 17 are explained in more
> detail in the sections below.
That is just a place-holder. I changed the bullet text to be:
TO BE COMPLETED LATER
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 04:44 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
@ 2024-05-09 13:08 ` Bruce Momjian <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 13:08 UTC (permalink / raw)
To: David Rowley <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 04:44:47PM +1200, David Rowley wrote:
> On Thu, 9 May 2024 at 16:04, Bruce Momjian <[email protected]> wrote:
> > I welcome feedback. For some reason it was an easier job than usual.
>
> Thanks for working on that.
>
> > +2023-11-02 [cac169d68] Increase DEFAULT_FDW_TUPLE_COST from 0.01 to 0.2
>
> > +Double the default foreign data wrapper tuple cost (David Rowley, Umair Shahid)
>
> That's 20x rather than 2x.
Oops, changed to:
Increase the default foreign data wrapper tuple cost (David
Rowley, Umair Shahid)
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-09 04:53 ` Bertrand Drouvot <[email protected]>
2024-05-09 13:29 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Bertrand Drouvot @ 2024-05-09 04:53 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
Hi,
On Thu, May 09, 2024 at 12:03:50AM -0400, Bruce Momjian wrote:
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
Thanks for working on that!
> I welcome feedback.
> Add system view pg_wait_events that reports wait event types (Michael Paquier)
Michael is the committer for 1e68e43d3f, the author is me.
Regards,
--
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 04:53 ` Re: First draft of PG 17 release notes Bertrand Drouvot <[email protected]>
@ 2024-05-09 13:29 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 13:29 UTC (permalink / raw)
To: Bertrand Drouvot <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 04:53:38AM +0000, Bertrand Drouvot wrote:
> Hi,
>
> On Thu, May 09, 2024 at 12:03:50AM -0400, Bruce Momjian wrote:
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
>
> Thanks for working on that!
>
> > I welcome feedback.
>
> > Add system view pg_wait_events that reports wait event types (Michael Paquier)
>
> Michael is the committer for 1e68e43d3f, the author is me.
Wow, thank you for finding that. The commit message is very clear so I
don't know how I made that mistake. Fixed.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-09 05:17 ` Masahiko Sawada <[email protected]>
2024-05-09 13:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Masahiko Sawada @ 2024-05-09 05:17 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
Hi,
On Thu, May 9, 2024 at 1:03 PM Bruce Momjian <[email protected]> wrote:
>
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
Thank you for working on that!
I'd like to mention some of my works. I think we can add the vacuum
performance improvements by the following commits:
- Add template for adaptive radix tree (ee1b30f1)
- Add TIDStore, to store sets of TIDs (ItemPointerData) efficiently (30e144287)
- Use TidStore for dead tuple TIDs storage during lazy vacuum (667e65aac)
Also, please consider the following item:
- Improve eviction algorithm in ReorderBuffer using max-heap for many
subtransactions (5bec1d6bc)
Finally, should we mention the following commit in the release note?
It's not a user-visible change but added a new regression test module.
- Add tests for XID wraparound (e255b646a)
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 05:17 ` Re: First draft of PG 17 release notes Masahiko Sawada <[email protected]>
@ 2024-05-09 13:48 ` Bruce Momjian <[email protected]>
2024-05-15 01:10 ` Re: First draft of PG 17 release notes Masahiko Sawada <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 13:48 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 02:17:12PM +0900, Masahiko Sawada wrote:
> Hi,
>
> On Thu, May 9, 2024 at 1:03 PM Bruce Momjian <[email protected]> wrote:
> >
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
>
> Thank you for working on that!
>
> I'd like to mention some of my works. I think we can add the vacuum
> performance improvements by the following commits:
>
> - Add template for adaptive radix tree (ee1b30f1)
> - Add TIDStore, to store sets of TIDs (ItemPointerData) efficiently (30e144287)
> - Use TidStore for dead tuple TIDs storage during lazy vacuum (667e65aac)
Okay, I reworded the item, added authors, and added the commits:
<!--
Author: John Naylor <[email protected]>
2024-03-07 [ee1b30f12] Add template for adaptive radix tree
Author: Masahiko Sawada <[email protected]>
2024-03-21 [30e144287] Add TIDStore, to store sets of TIDs (ItemPointerData) ef
Author: Masahiko Sawada <[email protected]>
2024-04-02 [667e65aac] Use TidStore for dead tuple TIDs storage during lazy vac
Author: Heikki Linnakangas <[email protected]>
2024-04-03 [6dbb49026] Combine freezing and pruning steps in VACUUM
-->
<listitem>
<para>
Allow vacuum to more efficiently remove and freeze tuples (John Naylor, Masahiko Sawada, Melanie Plageman)
</para>
</listitem>
> Also, please consider the following item:
>
> - Improve eviction algorithm in ReorderBuffer using max-heap for many
> subtransactions (5bec1d6bc)
I looked at that item and I don't have a generic "make logical
replication apply faster" item to merge it into, and many
subtransactions seemed like enough of an edge-case that I didn't think
mentioning it make sense. Can you see a good place to add it?
> Finally, should we mention the following commit in the release note?
> It's not a user-visible change but added a new regression test module.
>
> - Add tests for XID wraparound (e255b646a)
I don't normally add testing infrastructure changes unless they are
major.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 05:17 ` Re: First draft of PG 17 release notes Masahiko Sawada <[email protected]>
2024-05-09 13:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-15 01:10 ` Masahiko Sawada <[email protected]>
2024-05-15 02:20 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Masahiko Sawada @ 2024-05-15 01:10 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 10:48 PM Bruce Momjian <[email protected]> wrote:
>
> On Thu, May 9, 2024 at 02:17:12PM +0900, Masahiko Sawada wrote:
> > Hi,
> >
>
> > Also, please consider the following item:
> >
> > - Improve eviction algorithm in ReorderBuffer using max-heap for many
> > subtransactions (5bec1d6bc)
>
> I looked at that item and I don't have a generic "make logical
> replication apply faster" item to merge it into, and many
> subtransactions seemed like enough of an edge-case that I didn't think
> mentioning it make sense. Can you see a good place to add it?
I think that since many subtransactions cases are no longer becoming
edge-cases these days, we needed to improve that and it might be
helpful for users to mention it. How about the following item for
example?
Improve logical decoding performance in cases where there are many
subtransactions.
>
> > Finally, should we mention the following commit in the release note?
> > It's not a user-visible change but added a new regression test module.
> >
> > - Add tests for XID wraparound (e255b646a)
>
> I don't normally add testing infrastructure changes unless they are
> major.
I've seen we had such item, for example in PG14 release note:
Add a test module for the regular expression package (Tom Lane)
But if our policy has already changed, I'm okay with not mentioning
the xid_wraparound test in the PG17 release note.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 05:17 ` Re: First draft of PG 17 release notes Masahiko Sawada <[email protected]>
2024-05-09 13:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 01:10 ` Re: First draft of PG 17 release notes Masahiko Sawada <[email protected]>
@ 2024-05-15 02:20 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-15 02:20 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: pgsql-hackers
On Wed, May 15, 2024 at 10:10:28AM +0900, Masahiko Sawada wrote:
> > I looked at that item and I don't have a generic "make logical
> > replication apply faster" item to merge it into, and many
> > subtransactions seemed like enough of an edge-case that I didn't think
> > mentioning it make sense. Can you see a good place to add it?
>
> I think that since many subtransactions cases are no longer becoming
> edge-cases these days, we needed to improve that and it might be
> helpful for users to mention it. How about the following item for
> example?
>
> Improve logical decoding performance in cases where there are many
> subtransactions.
Okay, item added in the attached applied patch.
> > > Finally, should we mention the following commit in the release note?
> > > It's not a user-visible change but added a new regression test module.
> > >
> > > - Add tests for XID wraparound (e255b646a)
> >
> > I don't normally add testing infrastructure changes unless they are
> > major.
>
> I've seen we had such item, for example in PG14 release note:
>
> Add a test module for the regular expression package (Tom Lane)
>
> But if our policy has already changed, I'm okay with not mentioning
> the xid_wraparound test in the PG17 release note.
Uh, that PG 14 test suite was huge and flushed out a lot of bugs, not
only in our regex code but I think in the TCL/Henry Spencer regex
library we inherited.
We add 10-40 tests every year, and how many do I mention in the release
notes? You had to go back to PG 14 to find one. We have not changed
our release note "test item" criteria --- I only mention tests that are
significant to our userbase. I think that test suite was significant to
anyone using the TCL/Henry Spencer regex library.
If you want your test mentioned, you have to explain why it is useful
for users to know about it, or the value it brings them.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (713B, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 510810b53dc..e68c499e0db 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -1150,6 +1150,17 @@ Previously only btree indexes could be used for this purpose.
</para>
</listitem>
+<!--
+Author: Masahiko Sawada <[email protected]>
+2024-04-03 [5bec1d6bc] Improve eviction algorithm in ReorderBuffer using max-he
+-->
+
+<listitem>
+<para>
+Improve logical decoding performance in cases where there are many subtransactions (Masahiko Sawada)
+</para>
+</listitem>
+
<!--
Author: Amit Kapila <[email protected]>
2023-10-17 [79243de13] Restart the apply worker if the privileges have been rev
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-09 06:37 ` Richard Guo <[email protected]>
2024-05-09 14:17 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Richard Guo @ 2024-05-09 06:37 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
Thanks for working on that.
For this item:
> Allow the optimizer to improve CTE plans by using the sort order of
> columns referenced in earlier CTE clauses (Jian Guo)
I think you mean a65724dfa. The author should be 'Richard Guo'.
And I'm wondering if it is more accurate to state it as "Allow the
optimizer to improve plans for the outer query by leveraging the sort
order of a CTE's output."
I think maybe a similar revision can be applied to the item just above
this one.
Thanks
Richard
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 06:37 ` Re: First draft of PG 17 release notes Richard Guo <[email protected]>
@ 2024-05-09 14:17 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 14:17 UTC (permalink / raw)
To: Richard Guo <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 02:37:57PM +0800, Richard Guo wrote:
>
> On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
>
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
>
> Thanks for working on that.
>
> For this item:
>
>
> Allow the optimizer to improve CTE plans by using the sort order of
> columns referenced in earlier CTE clauses (Jian Guo)
>
>
> I think you mean a65724dfa. The author should be 'Richard Guo'.
Wow the CTE item above it was done by Jian Guo. I probably copied the
text from the line above it, modified the description, but thought the
author's name was the same, but it was not. Fixed.
> And I'm wondering if it is more accurate to state it as "Allow the
> optimizer to improve plans for the outer query by leveraging the sort
> order of a CTE's output."
>
> I think maybe a similar revision can be applied to the item just above
> this one.
Okay, I went with this text:
<!--
Author: Tom Lane <[email protected]>
2023-11-17 [f7816aec2] Extract column statistics from CTE references, if possib
-->
<listitem>
<para>
Allow the optimizer to improve CTE plans by considering the statistics of columns referenced in earlier row output clauses (Jian Guo, Tom Lane)
</para>
</listitem>
<!--
Author: Tom Lane <[email protected]>
2024-03-26 [a65724dfa] Propagate pathkeys from CTEs up to the outer query.
-->
<listitem>
<para>
Allow the optimizer to improve CTE plans by considering the sort order of columns referenced in earlier row output clauses (Richard Guo)
</para>
</listitem>
I did not use "leveraging" because I am concerned non-native English
speakers might find the term confusing.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-09 09:18 ` Aleksander Alekseev <[email protected]>
2024-05-09 14:20 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Aleksander Alekseev @ 2024-05-09 09:18 UTC (permalink / raw)
To: pgsql-hackers; +Cc: Bruce Momjian <[email protected]>
Hi,
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
> It will be improved until the final release. The item count is 188,
> which is similar to recent releases:
Thanks for working on this.
I believe the part of the 64-bit XIDs patchset that was delivered in
PG17 is worth highlighting in "E.1.3.10. Source Code" section:
4ed8f0913bfd
2cdf131c46e6
5a1dfde8334b
a60b8a58f435
All this can probably be summarized as one bullet "Index SLRUs by
64-bit integers rather than by 32-bit ones" where the authors are:
Maxim Orlov, Aleksander Alekseev, Alexander Korotkov, Teodor Sigaev,
Nikita Glukhov, Pavel Borisov, Yura Sokolov.
--
Best regards,
Aleksander Alekseev
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 09:18 ` Re: First draft of PG 17 release notes Aleksander Alekseev <[email protected]>
@ 2024-05-09 14:20 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 14:20 UTC (permalink / raw)
To: Aleksander Alekseev <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 12:18:44PM +0300, Aleksander Alekseev wrote:
> Hi,
>
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
> >
> > It will be improved until the final release. The item count is 188,
> > which is similar to recent releases:
>
> Thanks for working on this.
>
> I believe the part of the 64-bit XIDs patchset that was delivered in
> PG17 is worth highlighting in "E.1.3.10. Source Code" section:
>
> 4ed8f0913bfd
> 2cdf131c46e6
> 5a1dfde8334b
> a60b8a58f435
>
> All this can probably be summarized as one bullet "Index SLRUs by
> 64-bit integers rather than by 32-bit ones" where the authors are:
> Maxim Orlov, Aleksander Alekseev, Alexander Korotkov, Teodor Sigaev,
> Nikita Glukhov, Pavel Borisov, Yura Sokolov.
Wow, I try to only list source code items that have some user-facing
impact, and I don't think these do. I do realize how important they are
though. This gets into the balance of mentioning items _users_ need to
know about, vs. important improvements that _we_ know about.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-09 10:00 ` jian he <[email protected]>
2024-05-09 14:49 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 02:22 ` Re: First draft of PG 17 release notes Tender Wang <[email protected]>
27 siblings, 2 replies; 194+ messages in thread
From: jian he @ 2024-05-09 10:00 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
>
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
another potential incompatibilities issue:
ALTER TABLE DROP PRIMARY KEY
see:
https://www.postgresql.org/message-id/202404181849.6frtmajobe27%40alvherre.pgsql
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:00 ` Re: First draft of PG 17 release notes jian he <[email protected]>
@ 2024-05-09 14:49 ` Bruce Momjian <[email protected]>
2024-05-09 18:40 ` Re: First draft of PG 17 release notes Álvaro Herrera <[email protected]>
1 sibling, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 14:49 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: pgsql-hackers; Álvaro Herrera <[email protected]>
On Thu, May 9, 2024 at 06:00:24PM +0800, jian he wrote:
> On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> >
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
> >
>
> another potential incompatibilities issue:
> ALTER TABLE DROP PRIMARY KEY
>
> see:
> https://www.postgresql.org/message-id/202404181849.6frtmajobe27%40alvherre.pgsql
I see it now, and I see Alvaro Herrera saying:
https://www.postgresql.org/message-id/202404181849.6frtmajobe27%40alvherre.pgsql
> I wonder is there any incompatibility issue, or do we need to say something
> about the new behavior when dropping a key column?
--> Umm, yeah, maybe we should document it in ALTER TABLE DROP PRIMARY KEY
--> and in the release notes to note the different behavior.
However, I don't see it mentioned as a release note item in the commit
message or mentioned in our docs. I suppose the release note text would
be:
Removing a PRIMARY KEY will remove the NOT NULL column specification
Previously the NOT NULL specification would be retained.
Do we have agreement that we want this release note item?
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:00 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 14:49 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-09 18:40 ` Álvaro Herrera <[email protected]>
2024-05-09 23:54 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Álvaro Herrera @ 2024-05-09 18:40 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: jian he <[email protected]>; pgsql-hackers
On 2024-May-09, Bruce Momjian wrote:
> However, I don't see it mentioned as a release note item in the commit
> message or mentioned in our docs. I suppose the release note text would
> be:
>
> Removing a PRIMARY KEY will remove the NOT NULL column specification
>
> Previously the NOT NULL specification would be retained.
>
> Do we have agreement that we want this release note item?
Yes. Maybe we want some others too (especially regarding inheritance,
but also regarding the way we handle the constraints internally), and
maybe in this one we want different wording. How about something like
this:
Removing a primary key constraint may change the nullability
characteristic of the columns that the primary key covered.
If explicit not-null constraints exist on the same column, then they
continue to be /known not nullable/; otherwise they become /possibly
nullable/.
This is largely based on the SQL standard's language of a column
descriptor having a "nullability characteristic", which for columns with
not-null or primary key constraints is "known not null". I don't think
we use those terms anywhere. I hope this isn't too confusing.
The standard's text on this, in section "4.13 Columns, fields, and
attributes", is
Every column has a nullability characteristic that indicates whether
the value from that column can be the null value. A nullability
characteristic is either known not nullable or possibly nullable.
--
Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:00 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 14:49 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 18:40 ` Re: First draft of PG 17 release notes Álvaro Herrera <[email protected]>
@ 2024-05-09 23:54 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 23:54 UTC (permalink / raw)
To: Álvaro Herrera <[email protected]>; +Cc: jian he <[email protected]>; pgsql-hackers
On Thu, May 9, 2024 at 08:40:00PM +0200, Álvaro Herrera wrote:
> On 2024-May-09, Bruce Momjian wrote:
>
> > However, I don't see it mentioned as a release note item in the commit
> > message or mentioned in our docs. I suppose the release note text would
> > be:
> >
> > Removing a PRIMARY KEY will remove the NOT NULL column specification
> >
> > Previously the NOT NULL specification would be retained.
> >
> > Do we have agreement that we want this release note item?
>
> Yes. Maybe we want some others too (especially regarding inheritance,
> but also regarding the way we handle the constraints internally), and
> maybe in this one we want different wording. How about something like
> this:
>
> Removing a primary key constraint may change the nullability
> characteristic of the columns that the primary key covered.
>
> If explicit not-null constraints exist on the same column, then they
> continue to be /known not nullable/; otherwise they become /possibly
> nullable/.
>
> This is largely based on the SQL standard's language of a column
> descriptor having a "nullability characteristic", which for columns with
> not-null or primary key constraints is "known not null". I don't think
> we use those terms anywhere. I hope this isn't too confusing.
Yes, it was confusing, partly because it is using wording we don't use,
and partly because it is talking about what can go into the column,
rather than the visible column restriction NOT NULL. I also think "may"
is too imprecise.
How about:
Removing a primary key will remove a column's NOT NULL constraint
if the constraint was added by the primary key
Previously such NOT NULL constraints would remain after a primary
key was removed. A column-level NOT NULL constraint would not be
emoved.
Here is the PG 16 output:
CREATE TABLE test ( x INT CONSTRAINT test_pkey PRIMARY KEY );
Table "public.test"
Column | Type | Collation | Nullable | Default
--------+---------+-----------+----------+---------
x | integer | | not null |
Indexes:
"test_pkey" PRIMARY KEY, btree (x)
CREATE TABLE test_with_not_null (x INT NOT NULL CONSTRAINT test_pkey_with_not_null PRIMARY KEY);
Table "public.test_with_not_null"
Column | Type | Collation | Nullable | Default
--------+---------+-----------+----------+---------
x | integer | | not null |
Indexes:
"test_pkey_with_not_null" PRIMARY KEY, btree (x)
ALTER TABLE test DROP CONSTRAINT test_pkey;
Table "public.test"
Column | Type | Collation | Nullable | Default
--------+---------+-----------+----------+---------
--> x | integer | | not null |
ALTER TABLE test_with_not_null DROP CONSTRAINT test_pkey_with_not_null;
Table "public.test_with_not_null"
Column | Type | Collation | Nullable | Default
--------+---------+-----------+----------+---------
--> x | integer | | not null |
Here is the output in PG 17:
CREATE TABLE test ( x INT CONSTRAINT test_pkey PRIMARY KEY );
Table "public.test"
Column | Type | Collation | Nullable | Default
--------+---------+-----------+----------+---------
x | integer | | not null |
Indexes:
"test_pkey" PRIMARY KEY, btree (x)
CREATE TABLE test_with_not_null (x INT NOT NULL CONSTRAINT test_pkey_with_not_null PRIMARY KEY);
Table "public.test_with_not_null"
Column | Type | Collation | Nullable | Default
--------+---------+-----------+----------+---------
x | integer | | not null |
Indexes:
"test_pkey_with_not_null" PRIMARY KEY, btree (x)
ALTER TABLE test DROP CONSTRAINT test_pkey;
Table "public.test"
Column | Type | Collation | Nullable | Default
--------+---------+-----------+----------+---------
--> x | integer | | |
ALTER TABLE test_with_not_null DROP CONSTRAINT test_pkey_with_not_null;
Table "public.test_with_not_null"
Column | Type | Collation | Nullable | Default
--------+---------+-----------+----------+---------
--> x | integer | | not null |
Notice that the table without a _column_ NOT NULL removes the NOT NULL
designation after removing the primary key only in PG 17.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:00 ` Re: First draft of PG 17 release notes jian he <[email protected]>
@ 2024-05-14 02:22 ` Tender Wang <[email protected]>
2024-05-15 02:02 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
1 sibling, 1 reply; 194+ messages in thread
From: Tender Wang @ 2024-05-14 02:22 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Bruce Momjian <[email protected]>; pgsql-hackers
jian he <[email protected]> 于2024年5月9日周四 18:00写道：
> On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> >
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
> >
>
> another potential incompatibilities issue:
> ALTER TABLE DROP PRIMARY KEY
>
> see:
>
> https://www.postgresql.org/message-id/202404181849.6frtmajobe27%40alvherre.pgsql
>
>
Since Alvaro has reverted all changes to not-null constraints, so will not
have potential incompatibilities issue.
--
Tender Wang
OpenPie: https://en.openpie.com/
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:00 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-14 02:22 ` Re: First draft of PG 17 release notes Tender Wang <[email protected]>
@ 2024-05-15 02:02 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-15 02:02 UTC (permalink / raw)
To: Tender Wang <[email protected]>; +Cc: jian he <[email protected]>; pgsql-hackers
On Tue, May 14, 2024 at 10:22:35AM +0800, Tender Wang wrote:
>
>
> jian he <[email protected]> 于2024年5月9日周四 18:00写道：
>
> On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> >
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
> >
>
> another potential incompatibilities issue:
> ALTER TABLE DROP PRIMARY KEY
>
> see:
> https://www.postgresql.org/message-id/
> 202404181849.6frtmajobe27%40alvherre.pgsql
>
>
>
> Since Alvaro has reverted all changes to not-null constraints, so will not have
> potential incompatibilities issue.
Agreed.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-09 10:22 ` Dagfinn Ilmari Mannsåker <[email protected]>
2024-05-09 10:31 ` Re: First draft of PG 17 release notes Dagfinn Ilmari Mannsåker <[email protected]>
2024-05-09 14:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 2 replies; 194+ messages in thread
From: Dagfinn Ilmari Mannsåker @ 2024-05-09 10:22 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
Bruce Momjian <[email protected]> writes:
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
My name is listed twice in the "Improve psql tab completion" item.
- ilmari
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:22 ` Re: First draft of PG 17 release notes Dagfinn Ilmari Mannsåker <[email protected]>
@ 2024-05-09 10:31 ` Dagfinn Ilmari Mannsåker <[email protected]>
2024-05-09 14:51 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
1 sibling, 1 reply; 194+ messages in thread
From: Dagfinn Ilmari Mannsåker @ 2024-05-09 10:31 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
Dagfinn Ilmari Mannsåker <[email protected]> writes:
> Bruce Momjian <[email protected]> writes:
>
>> I have committed the first draft of the PG 17 release notes; you can
>> see the results here:
>>
>> https://momjian.us/pgsql_docs/release-17.html
>
> My name is listed twice in the "Improve psql tab completion" item.
You can move one of them to "Track DEALLOCATE in pg_stat_statements",
which Michael and I co-authored.
- ilmari
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:22 ` Re: First draft of PG 17 release notes Dagfinn Ilmari Mannsåker <[email protected]>
2024-05-09 10:31 ` Re: First draft of PG 17 release notes Dagfinn Ilmari Mannsåker <[email protected]>
@ 2024-05-09 14:51 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 14:51 UTC (permalink / raw)
To: Dagfinn Ilmari Mannsåker <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 11:31:17AM +0100, Dagfinn Ilmari Mannsåker wrote:
> Dagfinn Ilmari Mannsåker <[email protected]> writes:
>
> > Bruce Momjian <[email protected]> writes:
> >
> >> I have committed the first draft of the PG 17 release notes; you can
> >> see the results here:
> >>
> >> https://momjian.us/pgsql_docs/release-17.html
> >
> > My name is listed twice in the "Improve psql tab completion" item.
>
> You can move one of them to "Track DEALLOCATE in pg_stat_statements",
> which Michael and I co-authored.
Yep, also my mistake, fixed. My apologies.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:22 ` Re: First draft of PG 17 release notes Dagfinn Ilmari Mannsåker <[email protected]>
@ 2024-05-09 14:50 ` Bruce Momjian <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 14:50 UTC (permalink / raw)
To: Dagfinn Ilmari Mannsåker <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 11:22:06AM +0100, Dagfinn Ilmari Mannsåker wrote:
> Bruce Momjian <[email protected]> writes:
>
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
>
> My name is listed twice in the "Improve psql tab completion" item.
You did such a great job I wanted to list you twice. :-) Actually, the
author list was so long I just didn't notice, fixed.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-09 10:53 ` jian he <[email protected]>
2024-05-09 10:57 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 11:49 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 15:08 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 3 replies; 194+ messages in thread
From: jian he @ 2024-05-09 10:53 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
* Add function pg_buffercache_evict() to allow shared buffer eviction
(Palak Chaturvedi, Thomas Munro)
* This is useful for testing.
this should put it on the section
< E.1.3.11. Additional Modules
?
Then I found out official release notes don't have <section> attributes,
so it doesn't matter?
<<
Allow ALTER OPERATOR to set more optimization attributes (Tommy Pavlicek)
This is useful for extensions.
<<
I think this commit title "Add hash support functions and hash opclass
for contrib/ltree."
from [1] is more descriptive.
i am not 100% sure of the meaning of "This is useful for extensions."
[1] https://git.postgresql.org/cgit/postgresql.git/commit/?id=485f0aa85995340fb62113448c992ee48dc6fff1
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:53 ` Re: First draft of PG 17 release notes jian he <[email protected]>
@ 2024-05-09 10:57 ` jian he <[email protected]>
2024-05-09 15:09 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2 siblings, 1 reply; 194+ messages in thread
From: jian he @ 2024-05-09 10:57 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
> <<
> Allow ALTER OPERATOR to set more optimization attributes (Tommy Pavlicek)
> This is useful for extensions.
> <<
sorry, I mean
<<
Allow the creation of hash indexes on ltree columns (Tommy Pavlicek)
This also enables hash join and hash aggregation on ltree columns.
<<
better description would be:
<<
Add hash support functions and hash opclass for contrib/ltree (Tommy Pavlicek)
This also enables hash join and hash aggregation on ltree columns.
<<
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:53 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 10:57 ` Re: First draft of PG 17 release notes jian he <[email protected]>
@ 2024-05-09 15:09 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 15:09 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 06:57:01PM +0800, jian he wrote:
> > <<
> > Allow ALTER OPERATOR to set more optimization attributes (Tommy Pavlicek)
> > This is useful for extensions.
> > <<
>
> sorry, I mean
> <<
> Allow the creation of hash indexes on ltree columns (Tommy Pavlicek)
> This also enables hash join and hash aggregation on ltree columns.
> <<
>
> better description would be:
> <<
> Add hash support functions and hash opclass for contrib/ltree (Tommy Pavlicek)
> This also enables hash join and hash aggregation on ltree columns.
> <<
Yes, please see my previous email where I am asking why being more
specific is worse.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:53 ` Re: First draft of PG 17 release notes jian he <[email protected]>
@ 2024-05-09 11:49 ` jian he <[email protected]>
2024-05-09 15:12 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2 siblings, 1 reply; 194+ messages in thread
From: jian he @ 2024-05-09 11:49 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 6:53 PM jian he <[email protected]> wrote:
>
> On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
< Add columns to pg_stats to report range histogram information (Egor
Rogov, Soumyadeep Chakraborty)
I think this applies to range type and multi range type, "range
histogram information" seems not very clear to me.
So maybe:
< Add columns to pg_stats to report range-type histogram information
(Egor Rogov, Soumyadeep Chakraborty)
Display length and bounds histograms in pg_stats
< Add new COPY option "ON_ERROR ignore" to discard error rows (Damir
Belyalov, Atsushi Torikoshi, Alex Shulgin, Jian He, Jian He, Yugo
Nagata)
duplicate name.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:53 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 11:49 ` Re: First draft of PG 17 release notes jian he <[email protected]>
@ 2024-05-09 15:12 ` Bruce Momjian <[email protected]>
2024-05-09 15:26 ` Re: First draft of PG 17 release notes jian he <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 15:12 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 07:49:55PM +0800, jian he wrote:
> On Thu, May 9, 2024 at 6:53 PM jian he <[email protected]> wrote:
> >
> > On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> > > I have committed the first draft of the PG 17 release notes; you can
> > > see the results here:
> > >
> > > https://momjian.us/pgsql_docs/release-17.html
>
> < Add columns to pg_stats to report range histogram information (Egor
> Rogov, Soumyadeep Chakraborty)
> I think this applies to range type and multi range type, "range
> histogram information" seems not very clear to me.
> So maybe:
> < Add columns to pg_stats to report range-type histogram information
> (Egor Rogov, Soumyadeep Chakraborty)
Yes, good point, done.
> Display length and bounds histograms in pg_stats
Uh, isn't that assumed? Is this a detail worth mentioning?
> < Add new COPY option "ON_ERROR ignore" to discard error rows (Damir
> Belyalov, Atsushi Torikoshi, Alex Shulgin, Jian He, Jian He, Yugo
> Nagata)
> duplicate name.
Fixed.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:53 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 11:49 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 15:12 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-09 15:26 ` jian he <[email protected]>
2024-05-09 15:41 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: jian he @ 2024-05-09 15:26 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 11:12 PM Bruce Momjian <[email protected]> wrote:
>
> On Thu, May 9, 2024 at 07:49:55PM +0800, jian he wrote:
> > On Thu, May 9, 2024 at 6:53 PM jian he <[email protected]> wrote:
> > >
> > > On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> > > > I have committed the first draft of the PG 17 release notes; you can
> > > > see the results here:
> > > >
> > > > https://momjian.us/pgsql_docs/release-17.html
> >
E.1.3.1.5. Privileges
Add per-table GRANT permission MAINTAIN to control maintenance
operations (Nathan Bossart)
The operations are VACUUM, ANALYZE, REINDEX, REFRESH MATERIALIZE VIEW,
CLUSTER, and LOCK TABLE.
Add user-grantable role pg_maintain to control maintenance operations
(Nathan Bossart)
The operations are VACUUM, ANALYZE, REINDEX, REFRESH MATERIALIZE VIEW,
CLUSTER, and LOCK TABLE.
Allow roles with pg_monitor privileges to execute pg_current_logfile()
(Pavlo Golub, Nathan Bossart)
---------------
should be "REFRESH MATERIALIZED VIEW"?
also
"Allow roles with pg_monitor privileges to execute
pg_current_logfile() (Pavlo Golub, Nathan Bossart)"
"pg_monitor" is a predefined role, so technically, "with pg_monitor
privileges" is not correct?
--------------------------------------------------------------------------
Add function XMLText() to convert text to a single XML text node (Jim Jones)
XMLText()
should be
xmltext()
--------------------------------------------------------------------------
Add function to_regtypemod() to return the typemod of a string (David
Wheeler, Erik Wienhold)
I think this description does not mean the same thing as the doc[1]
[1] https://www.postgresql.org/docs/devel/functions-info.html#FUNCTIONS-INFO-CATALOG
--------------------------------------------------------------------------
Allow GROUP BY columns to be internally ordered to match ORDER BY
(Andrei Lepikhov, Teodor Sigaev)
This can be disabled using server variable enable_group_by_reordering.
Probably
`This can be disabled by setting the server variable
enable_group_by_reordering to false`.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:53 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 11:49 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 15:12 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 15:26 ` Re: First draft of PG 17 release notes jian he <[email protected]>
@ 2024-05-09 15:41 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 15:41 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 11:26:44PM +0800, jian he wrote:
> On Thu, May 9, 2024 at 11:12 PM Bruce Momjian <[email protected]> wrote:
> >
> > On Thu, May 9, 2024 at 07:49:55PM +0800, jian he wrote:
> > > On Thu, May 9, 2024 at 6:53 PM jian he <[email protected]> wrote:
> > > >
> > > > On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> > > > > I have committed the first draft of the PG 17 release notes; you can
> > > > > see the results here:
> > > > >
> > > > > https://momjian.us/pgsql_docs/release-17.html
> > >
>
> E.1.3.1.5. Privileges
> Add per-table GRANT permission MAINTAIN to control maintenance
> operations (Nathan Bossart)
>
> The operations are VACUUM, ANALYZE, REINDEX, REFRESH MATERIALIZE VIEW,
> CLUSTER, and LOCK TABLE.
>
> Add user-grantable role pg_maintain to control maintenance operations
> (Nathan Bossart)
>
> The operations are VACUUM, ANALYZE, REINDEX, REFRESH MATERIALIZE VIEW,
> CLUSTER, and LOCK TABLE.
>
> Allow roles with pg_monitor privileges to execute pg_current_logfile()
> (Pavlo Golub, Nathan Bossart)
> ---------------
> should be "REFRESH MATERIALIZED VIEW"?
Yes, fixed.
> also
> "Allow roles with pg_monitor privileges to execute
> pg_current_logfile() (Pavlo Golub, Nathan Bossart)"
> "pg_monitor" is a predefined role, so technically, "with pg_monitor
> privileges" is not correct?
Good point, new text:
Allow roles with pg_monitor membership to execute pg_current_logfile() (Pavlo Golub, Nathan Bossart)
> --------------------------------------------------------------------------
> Add function XMLText() to convert text to a single XML text node (Jim Jones)
>
> XMLText()
> should be
> xmltext()
Right, fixed.
> --------------------------------------------------------------------------
> Add function to_regtypemod() to return the typemod of a string (David
> Wheeler, Erik Wienhold)
> I think this description does not mean the same thing as the doc[1]
Yes, I see your point. I changed the text to:
Add function to_regtypemod() to return the type modifier of a
type specification (David Wheeler, Erik Wienhold)
> [1] https://www.postgresql.org/docs/devel/functions-info.html#FUNCTIONS-INFO-CATALOG
> --------------------------------------------------------------------------
>
> Allow GROUP BY columns to be internally ordered to match ORDER BY
> (Andrei Lepikhov, Teodor Sigaev)
> This can be disabled using server variable enable_group_by_reordering.
>
> Probably
> `This can be disabled by setting the server variable
> enable_group_by_reordering to false`.
Uh, I usually don't go into that detail. There will be a link to the
variable in about a month so users can look up its behavior.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:53 ` Re: First draft of PG 17 release notes jian he <[email protected]>
@ 2024-05-09 15:08 ` Bruce Momjian <[email protected]>
2 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 15:08 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 06:53:30PM +0800, jian he wrote:
> On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
> >
>
> * Add function pg_buffercache_evict() to allow shared buffer eviction
> (Palak Chaturvedi, Thomas Munro)
> * This is useful for testing.
>
> this should put it on the section
> < E.1.3.11. Additional Modules
> ?
Oh, it is in the pg_buffercache module --- I should have realized that
from the name, fixed.
> Then I found out official release notes don't have <section> attributes,
> so it doesn't matter?
Uh, what are sections? Did previous release notes have it?
> I think this commit title "Add hash support functions and hash opclass
> for contrib/ltree."
> from [1] is more descriptive.
Uh, I don't think people know what hash support functions are, but they
know what hash indexes are, and maybe hash joins and hash aggregates.
Why do you consider the commit text better?
> i am not 100% sure of the meaning of "This is useful for extensions."
The commit says:
commit 2b5154beab7
Author: Tom Lane <[email protected]>
Date: Fri Oct 20 12:28:38 2023 -0400
Extend ALTER OPERATOR to allow setting more optimization attributes.
Allow the COMMUTATOR, NEGATOR, MERGES, and HASHES attributes to be set
by ALTER OPERATOR. However, we don't allow COMMUTATOR/NEGATOR to be
changed once set, nor allow the MERGES/HASHES flags to be unset once
set. Changes like that might invalidate plans already made, and
dealing with the consequences seems like more trouble than it's worth.
--> The main use-case we foresee for this is to allow addition of missed
--> properties in extension update scripts, such as extending an existing
--> operator to support hashing. So only transitions from not-set to set
states seem very useful.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-09 16:10 ` Andrew Dunstan <[email protected]>
2024-05-09 16:29 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Andrew Dunstan @ 2024-05-09 16:10 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; pgsql-hackers
On 2024-05-09 Th 00:03, Bruce Momjian wrote:
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
> It will be improved until the final release. The item count is 188,
> which is similar to recent releases:
>
> release-10: 189
> release-11: 170
> release-12: 180
> release-13: 178
> release-14: 220
> release-15: 184
> release-16: 206
> release-17: 188
>
> I welcome feedback. For some reason it was an easier job than usual.
*
Remove the ability to build Postgres with Visual Studio (Michael
Paquier)
Meson is now the only available Windows build method.
This is a category mistake. What was removed was the special code we had
for building with VS, but not the ability to build with VS. You can
build with VS using meson (see for example drongo on the buildfarm)
cheers
andrew
--
Andrew Dunstan
EDB:https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 16:10 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
@ 2024-05-09 16:29 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 16:29 UTC (permalink / raw)
To: Andrew Dunstan <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 12:10:11PM -0400, Andrew Dunstan wrote:
>
> On 2024-05-09 Th 00:03, Bruce Momjian wrote:
>
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
> It will be improved until the final release. The item count is 188,
> which is similar to recent releases:
>
> release-10: 189
> release-11: 170
> release-12: 180
> release-13: 178
> release-14: 220
> release-15: 184
> release-16: 206
> release-17: 188
>
> I welcome feedback. For some reason it was an easier job than usual.
>
>
> • Remove the ability to build Postgres with Visual Studio (Michael Paquier)
>
> Meson is now the only available Windows build method.
>
>
> This is a category mistake. What was removed was the special code we had for
> building with VS, but not the ability to build with VS. You can build with VS
> using meson (see for example drongo on the buildfarm)
Wow, okay, I am not surprised I was confused. New text is:
<!--
Author: Michael Paquier <[email protected]>
2023-12-20 [1301c80b2] Remove MSVC scripts
-->
<listitem>
<para>
Remove the Microsoft Visual Studio Studio-specific Postgres build option (Michael Paquier)
</para>
<para>
Meson is now the only method for Visual Studio builds.
</para>
</listitem>
<!--
Author: Michael Paquier <[email protected]>
2023-12-20 [1301c80b2] Remove MSVC scripts
-->
<listitem>
<para>
Remove the Microsoft Visual Studio Studio-specific Postgres build option (Michael Paquier)
</para>
<para>
Meson is now the only method for Visual Studio builds.
</para>
</listitem>
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-09 20:05 ` Thomas Munro <[email protected]>
2024-05-09 20:35 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Thomas Munro @ 2024-05-09 20:05 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 4:04 PM Bruce Momjian <[email protected]> wrote:
> I welcome feedback. For some reason it was an easier job than usual.
> 2024-01-25 [820b5af73] jit: Require at least LLVM 10.
> Require LLVM version 10 or later (Peter Eisentraut)
Peter reviewed, I authored, and I think you intend to list authors in
parentheses.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 20:05 ` Re: First draft of PG 17 release notes Thomas Munro <[email protected]>
@ 2024-05-09 20:35 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-09 20:35 UTC (permalink / raw)
To: Thomas Munro <[email protected]>; +Cc: pgsql-hackers
On Fri, May 10, 2024 at 08:05:43AM +1200, Thomas Munro wrote:
> On Thu, May 9, 2024 at 4:04 PM Bruce Momjian <[email protected]> wrote:
> > I welcome feedback. For some reason it was an easier job than usual.
>
> > 2024-01-25 [820b5af73] jit: Require at least LLVM 10.
>
> > Require LLVM version 10 or later (Peter Eisentraut)
>
> Peter reviewed, I authored, and I think you intend to list authors in
> parentheses.
Yes, my mistake, fixed.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-10 08:24 ` Bharath Rupireddy <[email protected]>
2024-05-10 13:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Bharath Rupireddy @ 2024-05-10 08:24 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 9:34 AM Bruce Momjian <[email protected]> wrote:
>
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
> It will be improved until the final release. The item count is 188,
> which is similar to recent releases:
>
> release-10: 189
> release-11: 170
> release-12: 180
> release-13: 178
> release-14: 220
> release-15: 184
> release-16: 206
> release-17: 188
>
> I welcome feedback. For some reason it was an easier job than usual.
Thanks a lot for this work Bruce! It looks like commit
https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=91f2cae7a4e664e9c0472b364c7db29d7...
is missing from daft release notes. Just curious to know if it's
intentional or a miss out.
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 08:24 ` Re: First draft of PG 17 release notes Bharath Rupireddy <[email protected]>
@ 2024-05-10 13:50 ` Bruce Momjian <[email protected]>
2024-05-13 07:16 ` Re: First draft of PG 17 release notes Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-10 13:50 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: pgsql-hackers
On Fri, May 10, 2024 at 01:54:30PM +0530, Bharath Rupireddy wrote:
> On Thu, May 9, 2024 at 9:34 AM Bruce Momjian <[email protected]> wrote:
> >
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
> >
> > It will be improved until the final release. The item count is 188,
> > which is similar to recent releases:
> >
> > release-10: 189
> > release-11: 170
> > release-12: 180
> > release-13: 178
> > release-14: 220
> > release-15: 184
> > release-16: 206
> > release-17: 188
> >
> > I welcome feedback. For some reason it was an easier job than usual.
>
> Thanks a lot for this work Bruce! It looks like commit
> https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=91f2cae7a4e664e9c0472b364c7db29d7...
> is missing from daft release notes. Just curious to know if it's
> intentional or a miss out.
I did not mention it because the commit didn't mention any performance
benefit and it seemed more like an internal change than something people
needed to know about. I could reword and merge it into this item, if
you think I should:
Improve performance of heavily-contended WAL writes (Bharath Rupireddy)
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 08:24 ` Re: First draft of PG 17 release notes Bharath Rupireddy <[email protected]>
2024-05-10 13:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-13 07:16 ` Bharath Rupireddy <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bharath Rupireddy @ 2024-05-13 07:16 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Fri, May 10, 2024 at 7:20 PM Bruce Momjian <[email protected]> wrote:
>
> > Thanks a lot for this work Bruce! It looks like commit
> > https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=91f2cae7a4e664e9c0472b364c7db29d7...
> > is missing from daft release notes. Just curious to know if it's
> > intentional or a miss out.
>
> I did not mention it because the commit didn't mention any performance
> benefit and it seemed more like an internal change than something people
> needed to know about.
Yes, it's an internal feature for someone not using Direct IO for WAL
and helps achieve things mentioned at
https://www.postgresql.org/message-id/flat/20230125211540.zylu74dj2uuh3k7w%40awork3.anarazel.de#0cac...
(I'm hoping to target them for PG18). It starts to show visible
benefits if someone enables direct IO for WAL (for whatever reasons)
https://www.postgresql.org/message-id/CALj2ACV6rS%2B7iZx5%2BoAvyXJaN4AG-djAQeM1mrM%3DYSDkVrUs7g%40ma...
and https://www.postgresql.org/message-id/20230127061745.46yu4ksitzociwkt%40awork3.anarazel.de.
I'm okay if 91f2cae7 is left out for the reason that Direct IO for WAL
isn't something used in production and debug_io_direct is a developer
option.
> I could reword and merge it into this item, if
> you think I should:
>
> Improve performance of heavily-contended WAL writes (Bharath Rupireddy)
I think both the commits are for different purposes - one is for WAL
wrties, another is for WAL reads.
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-10 16:29 ` Daniel Verite <[email protected]>
2024-05-10 19:47 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Daniel Verite @ 2024-05-10 16:29 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
Bruce Momjian wrote:
> have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
In the psql items, I'd suggest mentioning
https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=90f5178
For the short description, maybe something like that:
- Improve FETCH_COUNT to work with all queries (Daniel Vérité)
Previously, results would be fetched in chunks only for queries
that start with the SELECT keyword.
Best regards,
--
Daniel Vérité
https://postgresql.verite.pro/
Twitter: @DanielVerite
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 16:29 ` Re: First draft of PG 17 release notes Daniel Verite <[email protected]>
@ 2024-05-10 19:47 ` Bruce Momjian <[email protected]>
2024-05-10 20:58 ` Re: First draft of PG 17 release notes Maiquel Grassi <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-10 19:47 UTC (permalink / raw)
To: Daniel Verite <[email protected]>; +Cc: pgsql-hackers
On Fri, May 10, 2024 at 06:29:11PM +0200, Daniel Verite wrote:
> Bruce Momjian wrote:
>
> > have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
>
> In the psql items, I'd suggest mentioning
>
> https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=90f5178
>
> For the short description, maybe something like that:
>
> - Improve FETCH_COUNT to work with all queries (Daniel Vérité)
> Previously, results would be fetched in chunks only for queries
> that start with the SELECT keyword.
Agreed, patch attached and applied.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (1.0K, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index e4b34d827d1..08238be9cb7 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -1893,8 +1893,6 @@ This is similar to PQpipelineSync() but it does not flush to the server unless t
<!--
Author: Tom Lane <[email protected]>
2024-04-06 [4643a2b26] Support retrieval of results in chunks with libpq.
-Author: Tom Lane <[email protected]>
-2024-04-06 [90f517821] Re-implement psql's FETCH_COUNT feature atop libpq's chu
-->
<listitem>
@@ -1937,6 +1935,17 @@ This is enabled with the client-side option sslnegotation=direct, requires ALPN,
<itemizedlist>
+<!--
+Author: Tom Lane <[email protected]>
+2024-04-06 [90f517821] Re-implement psql's FETCH_COUNT feature atop libpq's chu
+-->
+
+<listitem>
+<para>
+Allow FETCH_COUNT to work with non-SELECT queries (Daniel Vérité)
+</para>
+</listitem>
+
<!--
Author: Tom Lane <[email protected]>
2023-11-13 [d1379ebf4] Improve default and empty privilege outputs in psql.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 16:29 ` Re: First draft of PG 17 release notes Daniel Verite <[email protected]>
2024-05-10 19:47 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-10 20:58 ` Maiquel Grassi <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Maiquel Grassi @ 2024-05-10 20:58 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; Daniel Verite <[email protected]>; +Cc: pgsql-hackers
Mhd
Enviado desde Outlook para Android<https://aka.ms/AAb9ysg;
________________________________
From: Bruce Momjian <[email protected]>
Sent: Friday, May 10, 2024 4:47:04 PM
To: Daniel Verite <[email protected]>
Cc: PostgreSQL-development <[email protected]>
Subject: Re: First draft of PG 17 release notes
On Fri, May 10, 2024 at 06:29:11PM +0200, Daniel Verite wrote:
> Bruce Momjian wrote:
>
> > have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
>
> In the psql items, I'd suggest mentioning
>
> https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=90f5178
>
> For the short description, maybe something like that:
>
> - Improve FETCH_COUNT to work with all queries (Daniel Vérité)
> Previously, results would be fetched in chunks only for queries
> that start with the SELECT keyword.
Agreed, patch attached and applied.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-10 16:50 ` Jelte Fennema-Nio <[email protected]>
2024-05-10 21:21 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Jelte Fennema-Nio @ 2024-05-10 16:50 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, 9 May 2024 at 06:04, Bruce Momjian <[email protected]> wrote:
>
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
Great work!
There are two commits that I think would benefit from being listed
(but maybe they are already listed and I somehow missed them, or they
are left out on purpose for some reason):
- c4ab7da60617f020e8d75b1584d0754005d71830
- cafe1056558fe07cdc52b95205588fcd80870362
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 16:50 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
@ 2024-05-10 21:21 ` Bruce Momjian <[email protected]>
2024-05-10 21:31 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-10 21:21 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: pgsql-hackers
On Fri, May 10, 2024 at 06:50:54PM +0200, Jelte Fennema-Nio wrote:
> On Thu, 9 May 2024 at 06:04, Bruce Momjian <[email protected]> wrote:
> >
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
>
> Great work!
>
> There are two commits that I think would benefit from being listed
> (but maybe they are already listed and I somehow missed them, or they
> are left out on purpose for some reason):
I looked at both of these. In both cases I didn't see why the user
would need to know these changes were made:
---------------------------------------------------------------------------
> - c4ab7da60617f020e8d75b1584d0754005d71830
commit c4ab7da6061
Author: David Rowley <[email protected]>
Date: Sun Apr 7 21:20:18 2024 +1200
Avoid needless large memcpys in libpq socket writing
Until now, when calling pq_putmessage to write new data to a libpq
socket, all writes are copied into a buffer and that buffer gets flushed
when full to avoid having to perform small writes to the socket.
There are cases where we must write large amounts of data to the socket,
sometimes larger than the size of the buffer. In this case, it's
wasteful to memcpy this data into the buffer and flush it out, instead,
we can send it directly from the memory location that the data is already
stored in.
Here we adjust internal_putbytes() so that after having just flushed the
buffer to the socket, if the remaining bytes to send is as big or bigger
than the buffer size, we just send directly rather than needlessly
copying into the PqSendBuffer buffer first.
Examples of operations that write large amounts of data in one message
are; outputting large tuples with SELECT or COPY TO STDOUT and
pg_basebackup.
Author: Melih Mutlu
Reviewed-by: Heikki Linnakangas
Reviewed-by: Jelte Fennema-Nio
Reviewed-by: David Rowley
Reviewed-by: Ranier Vilela
Reviewed-by: Andres Freund
Discussion: https://postgr.es/m/CAGPVpCR15nosj0f6xe-c2h477zFR88q12e6WjEoEZc8ZYkTh3Q@mail.gmail.com
> - cafe1056558fe07cdc52b95205588fcd80870362
commit cafe1056558
Author: Robert Haas <[email protected]>
Date: Tue Apr 2 10:26:10 2024 -0400
Allow SIGINT to cancel psql database reconnections.
After installing the SIGINT handler in psql, SIGINT can no longer cancel
database reconnections. For instance, if the user starts a reconnection
and then needs to do some form of interaction (ie psql is polling),
there is no way to cancel the reconnection process currently.
Use PQconnectStartParams() in order to insert a cancel_pressed check
into the polling loop.
Tristan Partin, reviewed by Gurjeet Singh, Heikki Linnakangas, Jelte
Fennema-Nio, and me.
Discussion: http://postgr.es/m/[email protected]
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 16:50 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-10 21:21 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-10 21:31 ` Tom Lane <[email protected]>
2024-05-10 21:37 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-11 13:57 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-14 00:30 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 3 replies; 194+ messages in thread
From: Tom Lane @ 2024-05-10 21:31 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; pgsql-hackers
Bruce Momjian <[email protected]> writes:
> On Fri, May 10, 2024 at 06:50:54PM +0200, Jelte Fennema-Nio wrote:
>> There are two commits that I think would benefit from being listed
>> (but maybe they are already listed and I somehow missed them, or they
>> are left out on purpose for some reason):
> I looked at both of these. In both cases I didn't see why the user
> would need to know these changes were made:
I agree that the buffering change is not likely interesting, but
the fact that you can now control-C out of a psql "\c" command
is user-visible. People might have internalized the fact that
it didn't work, or created complicated workarounds.
regards, tom lane
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 16:50 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-10 21:21 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 21:31 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
@ 2024-05-10 21:37 ` Bruce Momjian <[email protected]>
2 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-10 21:37 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Fri, May 10, 2024 at 05:31:33PM -0400, Tom Lane wrote:
> Bruce Momjian <[email protected]> writes:
> > On Fri, May 10, 2024 at 06:50:54PM +0200, Jelte Fennema-Nio wrote:
> >> There are two commits that I think would benefit from being listed
> >> (but maybe they are already listed and I somehow missed them, or they
> >> are left out on purpose for some reason):
>
> > I looked at both of these. In both cases I didn't see why the user
> > would need to know these changes were made:
>
> I agree that the buffering change is not likely interesting, but
> the fact that you can now control-C out of a psql "\c" command
> is user-visible. People might have internalized the fact that
> it didn't work, or created complicated workarounds.
It was not clear to me what the user-visible behavior was with the
SIGINT control. Yes, based on your details, it should be mentioned.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 16:50 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-10 21:21 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 21:31 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
@ 2024-05-11 13:57 ` Jelte Fennema-Nio <[email protected]>
2024-05-11 14:24 ` Re: First draft of PG 17 release notes Joe Conway <[email protected]>
2 siblings, 1 reply; 194+ messages in thread
From: Jelte Fennema-Nio @ 2024-05-11 13:57 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Bruce Momjian <[email protected]>; pgsql-hackers
On Fri, 10 May 2024 at 23:31, Tom Lane <[email protected]> wrote:
>
> Bruce Momjian <[email protected]> writes:
> > I looked at both of these. In both cases I didn't see why the user
> > would need to know these changes were made:
>
> I agree that the buffering change is not likely interesting, but
> the fact that you can now control-C out of a psql "\c" command
> is user-visible. People might have internalized the fact that
> it didn't work, or created complicated workarounds.
The buffering change improved performance up to ~40% in some of the
benchmarks. The case it improves mostly is COPY of large rows and
streaming a base backup. That sounds user-visible enough to me to
warrant an entry imho.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 16:50 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-10 21:21 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 21:31 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
2024-05-11 13:57 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
@ 2024-05-11 14:24 ` Joe Conway <[email protected]>
2024-05-14 00:56 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Joe Conway @ 2024-05-11 14:24 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; Tom Lane <[email protected]>; +Cc: Bruce Momjian <[email protected]>; pgsql-hackers
On 5/11/24 09:57, Jelte Fennema-Nio wrote:
> On Fri, 10 May 2024 at 23:31, Tom Lane <[email protected]> wrote:
>>
>> Bruce Momjian <[email protected]> writes:
>> > I looked at both of these. In both cases I didn't see why the user
>> > would need to know these changes were made:
>>
>> I agree that the buffering change is not likely interesting, but
>> the fact that you can now control-C out of a psql "\c" command
>> is user-visible. People might have internalized the fact that
>> it didn't work, or created complicated workarounds.
>
> The buffering change improved performance up to ~40% in some of the
> benchmarks. The case it improves mostly is COPY of large rows and
> streaming a base backup. That sounds user-visible enough to me to
> warrant an entry imho.
+1
--
Joe Conway
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 16:50 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-10 21:21 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 21:31 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
2024-05-11 13:57 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-11 14:24 ` Re: First draft of PG 17 release notes Joe Conway <[email protected]>
@ 2024-05-14 00:56 ` Bruce Momjian <[email protected]>
2024-05-14 12:20 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-14 00:56 UTC (permalink / raw)
To: Joe Conway <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Tom Lane <[email protected]>; pgsql-hackers
On Sat, May 11, 2024 at 10:24:39AM -0400, Joe Conway wrote:
> On 5/11/24 09:57, Jelte Fennema-Nio wrote:
> > On Fri, 10 May 2024 at 23:31, Tom Lane <[email protected]> wrote:
> > >
> > > Bruce Momjian <[email protected]> writes:
> > > > I looked at both of these. In both cases I didn't see why the user
> > > > would need to know these changes were made:
> > >
> > > I agree that the buffering change is not likely interesting, but
> > > the fact that you can now control-C out of a psql "\c" command
> > > is user-visible. People might have internalized the fact that
> > > it didn't work, or created complicated workarounds.
> >
> > The buffering change improved performance up to ~40% in some of the
> > benchmarks. The case it improves mostly is COPY of large rows and
> > streaming a base backup. That sounds user-visible enough to me to
> > warrant an entry imho.
>
> +1
Attached patch applied.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (1.1K, ../../[email protected]/2-master.diff)
download | inline diff:
commit e87e7324555
Author: Bruce Momjian <[email protected]>
Date: Mon May 13 20:55:13 2024 -0400
doc PG 17 relnotes: add item about libpq large data transfers
Reported-by: Jelte Fennema-Nio
Discussion: https://postgr.es/m/CAGECzQTz5aUqLEL6daLd2Hu2FXS_LOSh4keDndJ1fwThsb_b5w@mail.gmail.com
Reviewed-by: Joe Conway
Backpatch-through: master
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 9dd3954f3c2..38c14970822 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -1901,6 +1901,17 @@ Add libpq function PQsetChunkedRowsMode to allow retrieval of results in chunks
</para>
</listitem>
+<!--
+Author: David Rowley <[email protected]>
+2024-04-07 [c4ab7da60] Avoid needless large memcpys in libpq socket writing
+-->
+
+<listitem>
+<para>
+Allow libpq to more efficiently transfer large blocks of data (Melih Mutlu)
+</para>
+</listitem>
+
<!--
Author: Heikki Linnakangas <[email protected]>
2024-04-08 [d39a49c1e] Support TLS handshake directly without SSLRequest negoti
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 16:50 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-10 21:21 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 21:31 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
2024-05-11 13:57 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-11 14:24 ` Re: First draft of PG 17 release notes Joe Conway <[email protected]>
2024-05-14 00:56 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-14 12:20 ` Jelte Fennema-Nio <[email protected]>
2024-05-15 00:47 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Jelte Fennema-Nio @ 2024-05-14 12:20 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Joe Conway <[email protected]>; Tom Lane <[email protected]>; pgsql-hackers
On Tue, 14 May 2024 at 02:56, Bruce Momjian <[email protected]> wrote:
>
> On Sat, May 11, 2024 at 10:24:39AM -0400, Joe Conway wrote:
> > On 5/11/24 09:57, Jelte Fennema-Nio wrote:
> > > The buffering change improved performance up to ~40% in some of the
> > > benchmarks. The case it improves mostly is COPY of large rows and
> > > streaming a base backup. That sounds user-visible enough to me to
> > > warrant an entry imho.
> >
> > +1
>
> Attached patch applied.
I think we shouldn't list this under the libpq changes and shouldn't
mention libpq in the description, since this patch changes
src/backend/libpq files instead of src/interfaces/libpq files. I think
it should be in the "General performance" section and describe the
change as something like the below:
Improve performance when transferring large blocks of data to a client
PS. I completely understand that this was not clear from the commit message.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 16:50 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-10 21:21 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 21:31 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
2024-05-11 13:57 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-11 14:24 ` Re: First draft of PG 17 release notes Joe Conway <[email protected]>
2024-05-14 00:56 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 12:20 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
@ 2024-05-15 00:47 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-15 00:47 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Joe Conway <[email protected]>; Tom Lane <[email protected]>; pgsql-hackers
On Tue, May 14, 2024 at 02:20:24PM +0200, Jelte Fennema-Nio wrote:
> On Tue, 14 May 2024 at 02:56, Bruce Momjian <[email protected]> wrote:
> >
> > On Sat, May 11, 2024 at 10:24:39AM -0400, Joe Conway wrote:
> > > On 5/11/24 09:57, Jelte Fennema-Nio wrote:
> > > > The buffering change improved performance up to ~40% in some of the
> > > > benchmarks. The case it improves mostly is COPY of large rows and
> > > > streaming a base backup. That sounds user-visible enough to me to
> > > > warrant an entry imho.
> > >
> > > +1
> >
> > Attached patch applied.
>
> I think we shouldn't list this under the libpq changes and shouldn't
> mention libpq in the description, since this patch changes
> src/backend/libpq files instead of src/interfaces/libpq files. I think
> it should be in the "General performance" section and describe the
> change as something like the below:
>
> Improve performance when transferring large blocks of data to a client
>
> PS. I completely understand that this was not clear from the commit message.
Okay, I went with your wording. Attached patch applied.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (1.2K, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index a3b4a8fb3b9..448f5653bc5 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -563,6 +563,17 @@ Improve performance of heavily-contended WAL writes (Bharath Rupireddy)
</para>
</listitem>
+<!--
+Author: David Rowley <[email protected]>
+2024-04-07 [c4ab7da60] Avoid needless large memcpys in libpq socket writing
+-->
+
+<listitem>
+<para>
+Improve performance when transferring large blocks of data to a client (Melih Mutlu)
+</para>
+</listitem>
+
<!--
Author: Thomas Munro <[email protected]>
2024-04-03 [210622c60] Provide vectored variant of ReadBuffer().
@@ -1923,17 +1934,6 @@ Add libpq function PQsetChunkedRowsMode to allow retrieval of results in chunks
</para>
</listitem>
-<!--
-Author: David Rowley <[email protected]>
-2024-04-07 [c4ab7da60] Avoid needless large memcpys in libpq socket writing
--->
-
-<listitem>
-<para>
-Allow libpq to more efficiently transfer large blocks of data (Melih Mutlu)
-</para>
-</listitem>
-
<!--
Author: Heikki Linnakangas <[email protected]>
2024-04-08 [d39a49c1e] Support TLS handshake directly without SSLRequest negoti
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 16:50 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-10 21:21 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 21:31 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
@ 2024-05-14 00:30 ` Bruce Momjian <[email protected]>
2 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-14 00:30 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; pgsql-hackers
On Fri, May 10, 2024 at 05:31:33PM -0400, Tom Lane wrote:
> Bruce Momjian <[email protected]> writes:
> > On Fri, May 10, 2024 at 06:50:54PM +0200, Jelte Fennema-Nio wrote:
> >> There are two commits that I think would benefit from being listed
> >> (but maybe they are already listed and I somehow missed them, or they
> >> are left out on purpose for some reason):
>
> > I looked at both of these. In both cases I didn't see why the user
> > would need to know these changes were made:
>
> I agree that the buffering change is not likely interesting, but
> the fact that you can now control-C out of a psql "\c" command
> is user-visible. People might have internalized the fact that
> it didn't work, or created complicated workarounds.
Agreed, attached patch applied.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (630B, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 5a741ff16f1..9dd3954f3c2 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -1980,6 +1980,17 @@ The parameter is "min_rows".
</para>
</listitem>
+<!--
+Author: Robert Haas <[email protected]>
+2024-04-02 [cafe10565] Allow SIGINT to cancel psql database reconnections.
+-->
+
+<listitem>
+<para>
+Allow psql connections to be canceled with control-C (Tristan Partin)
+</para>
+</listitem>
+
<!--
Author: Tom Lane <[email protected]>
2024-04-06 [90f517821] Re-implement psql's FETCH_COUNT feature atop libpq's chu
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-11 05:27 ` Andy Fan <[email protected]>
2024-05-11 05:57 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-14 00:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 2 replies; 194+ messages in thread
From: Andy Fan @ 2024-05-11 05:27 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers; [email protected]
Hello Bruce,
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
Thank you for working on this!
> I welcome feedback. For some reason it was an easier job than usual.
Do you think we need to add the following 2 items?
- 9f133763961e280d8ba692bcad0b061b861e9138 this is an optimizer
transform improvement.
- a8a968a8212ee3ef7f22795c834b33d871fac262 this is an optimizer costing
improvement.
Both of them can generate a better plan on some cases.
--
Best Regards
Andy Fan
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-11 05:27 ` Re: First draft of PG 17 release notes Andy Fan <[email protected]>
@ 2024-05-11 05:57 ` David Rowley <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: David Rowley @ 2024-05-11 05:57 UTC (permalink / raw)
To: Andy Fan <[email protected]>; +Cc: Bruce Momjian <[email protected]>; pgsql-hackers; [email protected]
On Sat, 11 May 2024 at 17:32, Andy Fan <[email protected]> wrote:
> Do you think we need to add the following 2 items?
>
> - 9f133763961e280d8ba692bcad0b061b861e9138 this is an optimizer
> transform improvement.
I think this should be in the release notes.
Suggest:
* Allow correlated IN subqueries to be transformed into joins (Andy
Fan, Tom Lane)
> - a8a968a8212ee3ef7f22795c834b33d871fac262 this is an optimizer costing
> improvement.
>
> Both of them can generate a better plan on some cases.
I think this should be present too.
Suggest:
* Improve optimizer's ability to use cheap startup plans when querying
partitioned tables, inheritance parents and for UNION ALL (Andy Fan,
David Rowley)
Both under "E.1.3.1.1. Optimizer"
David
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-11 05:27 ` Re: First draft of PG 17 release notes Andy Fan <[email protected]>
@ 2024-05-14 00:59 ` Bruce Momjian <[email protected]>
2024-05-14 02:32 ` Re: First draft of PG 17 release notes Andy Fan <[email protected]>
1 sibling, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-14 00:59 UTC (permalink / raw)
To: Andy Fan <[email protected]>; +Cc: pgsql-hackers; [email protected]
On Sat, May 11, 2024 at 01:27:25PM +0800, Andy Fan wrote:
>
> Hello Bruce,
>
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
>
> Thank you for working on this!
>
> > I welcome feedback. For some reason it was an easier job than usual.
>
> Do you think we need to add the following 2 items?
>
> - 9f133763961e280d8ba692bcad0b061b861e9138 this is an optimizer
> transform improvement.
It was unclear from the commit message exactly what user-visible
optimization this allowed. Do you have details?
> - a8a968a8212ee3ef7f22795c834b33d871fac262 this is an optimizer costing
> improvement.
Does this allow faster UNION ALL with LIMIT, perhaps?
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-11 05:27 ` Re: First draft of PG 17 release notes Andy Fan <[email protected]>
2024-05-14 00:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-14 02:32 ` Andy Fan <[email protected]>
2024-05-15 00:37 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Andy Fan @ 2024-05-14 02:32 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers; [email protected]
Bruce Momjian <[email protected]> writes:
> On Sat, May 11, 2024 at 01:27:25PM +0800, Andy Fan wrote:
>>
>> Hello Bruce,
>>
>> > I have committed the first draft of the PG 17 release notes; you can
>> > see the results here:
>> >
>> > https://momjian.us/pgsql_docs/release-17.html
>>
>> Thank you for working on this!
>>
>> > I welcome feedback. For some reason it was an easier job than usual.
>>
>> Do you think we need to add the following 2 items?
>>
>> - 9f133763961e280d8ba692bcad0b061b861e9138 this is an optimizer
>> transform improvement.
>
> It was unclear from the commit message exactly what user-visible
> optimization this allowed. Do you have details?
Yes, It allows the query like "SELECT * FROM t1 WHERE t1.a in (SELECT a
FROM t2 WHERE t2.b = t1.b)" be pulled up a semi join, hence more join
methods / join orders are possible.
>
>> - a8a968a8212ee3ef7f22795c834b33d871fac262 this is an optimizer costing
>> improvement.
>
> Does this allow faster UNION ALL with LIMIT, perhaps?
Yes, for example: (subquery-1) UNION ALL (subquery-2) LIMIT n;
When planning the subquery-1 or subquery-2, limit N should be
considered. As a consequence, maybe hash join should be replaced with
Nested Loop. Before this commits, it is ignored if it is flatten into
appendrel, and the "flatten" happens very often.
David provided a summary for the both commits in [1].
[1]
https://www.postgresql.org/message-id/CAApHDvqAQgq27LgYmJ85VVGTR0%3DhRW6HHq2oZgK0ZiYC_a%2BEww%40mail...
--
Best Regards
Andy Fan
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-11 05:27 ` Re: First draft of PG 17 release notes Andy Fan <[email protected]>
2024-05-14 00:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 02:32 ` Re: First draft of PG 17 release notes Andy Fan <[email protected]>
@ 2024-05-15 00:37 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-15 00:37 UTC (permalink / raw)
To: Andy Fan <[email protected]>; +Cc: pgsql-hackers; [email protected]
On Tue, May 14, 2024 at 10:32:14AM +0800, Andy Fan wrote:
> Bruce Momjian <[email protected]> writes:
> > It was unclear from the commit message exactly what user-visible
> > optimization this allowed. Do you have details?
>
> Yes, It allows the query like "SELECT * FROM t1 WHERE t1.a in (SELECT a
> FROM t2 WHERE t2.b = t1.b)" be pulled up a semi join, hence more join
> methods / join orders are possible.
>
>
> Yes, for example: (subquery-1) UNION ALL (subquery-2) LIMIT n;
>
> When planning the subquery-1 or subquery-2, limit N should be
> considered. As a consequence, maybe hash join should be replaced with
> Nested Loop. Before this commits, it is ignored if it is flatten into
> appendrel, and the "flatten" happens very often.
>
> David provided a summary for the both commits in [1].
Okay, attached patch applied.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (1022B, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 38c14970822..fa0a703629c 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -356,6 +356,28 @@ Improve optimization of range values when using containment operators <@ and
</para>
</listitem>
+<!--
+Author: Alexander Korotkov <[email protected]>
+2024-02-15 [9f1337639] Pull up ANY-SUBLINK with the necessary lateral support.
+-->
+
+<listitem>
+<para>
+Allow correlated IN subqueries to be transformed into joins (Andy Fan, Tom Lane)
+</para>
+</listitem>
+
+<!--
+Author: David Rowley <[email protected]>
+2023-10-05 [a8a968a82] Consider cheap startup paths in add_paths_to_append_rel
+-->
+
+<listitem>
+<para>
+Improve optimization of the LIMIT clause on partitioned tables, inheritance parents, and UNION ALL queries (Andy Fan, David Rowley)
+</para>
+</listitem>
+
<!--
Author: Tom Lane <[email protected]>
2023-07-14 [e08d74ca1] Allow plan nodes with initPlans to be considered paralle
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-11 19:32 ` Andrew Dunstan <[email protected]>
2024-05-15 00:39 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Andrew Dunstan @ 2024-05-11 19:32 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; pgsql-hackers
On 2024-05-09 Th 00:03, Bruce Momjian wrote:
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
> It will be improved until the final release. The item count is 188,
> which is similar to recent releases:
>
> release-10: 189
> release-11: 170
> release-12: 180
> release-13: 178
> release-14: 220
> release-15: 184
> release-16: 206
> release-17: 188
>
> I welcome feedback. For some reason it was an easier job than usual.
I don't like blowing my own horn but I feel commit 3311ea86ed "Introduce
a non-recursive JSON parser" should be in the release notes. This isn't
something that's purely internal, but it could be used by an extension
or a client program to parse JSON documents that are too large to handle
with the existing API.
Maybe "Introduce an incremental JSON parser" would have been a better
headline.
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-11 19:32 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
@ 2024-05-15 00:39 ` Bruce Momjian <[email protected]>
2024-05-16 15:50 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-15 00:39 UTC (permalink / raw)
To: Andrew Dunstan <[email protected]>; +Cc: pgsql-hackers
On Sat, May 11, 2024 at 03:32:55PM -0400, Andrew Dunstan wrote:
>
> On 2024-05-09 Th 00:03, Bruce Momjian wrote:
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
> >
> > It will be improved until the final release. The item count is 188,
> > which is similar to recent releases:
> >
> > release-10: 189
> > release-11: 170
> > release-12: 180
> > release-13: 178
> > release-14: 220
> > release-15: 184
> > release-16: 206
> > release-17: 188
> >
> > I welcome feedback. For some reason it was an easier job than usual.
>
>
> I don't like blowing my own horn but I feel commit 3311ea86ed "Introduce a
> non-recursive JSON parser" should be in the release notes. This isn't
> something that's purely internal, but it could be used by an extension or a
> client program to parse JSON documents that are too large to handle with the
> existing API.
>
> Maybe "Introduce an incremental JSON parser" would have been a better
> headline.
Well, this gets into a level of detail that is beyond the average
reader. I think at that level people will need to read the git logs or
review the code. Do we use it for anything yet?
It could be put in the source code section but I try to only have
user-visible stuff there.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-11 19:32 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
2024-05-15 00:39 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-16 15:50 ` Andrew Dunstan <[email protected]>
2024-05-18 16:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Andrew Dunstan @ 2024-05-16 15:50 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On 2024-05-14 Tu 20:39, Bruce Momjian wrote:
> On Sat, May 11, 2024 at 03:32:55PM -0400, Andrew Dunstan wrote:
>> On 2024-05-09 Th 00:03, Bruce Momjian wrote:
>>> I have committed the first draft of the PG 17 release notes; you can
>>> see the results here:
>>>
>>> https://momjian.us/pgsql_docs/release-17.html
>>>
>>> It will be improved until the final release. The item count is 188,
>>> which is similar to recent releases:
>>>
>>> release-10: 189
>>> release-11: 170
>>> release-12: 180
>>> release-13: 178
>>> release-14: 220
>>> release-15: 184
>>> release-16: 206
>>> release-17: 188
>>>
>>> I welcome feedback. For some reason it was an easier job than usual.
>>
>> I don't like blowing my own horn but I feel commit 3311ea86ed "Introduce a
>> non-recursive JSON parser" should be in the release notes. This isn't
>> something that's purely internal, but it could be used by an extension or a
>> client program to parse JSON documents that are too large to handle with the
>> existing API.
>>
>> Maybe "Introduce an incremental JSON parser" would have been a better
>> headline.
> Well, this gets into a level of detail that is beyond the average
> reader. I think at that level people will need to read the git logs or
> review the code. Do we use it for anything yet?
Yes, certainly, it's used in handling backup manifests. Without it we
can't handle huge manifests. See commits ea7b4e9a2a and 222e11a10a.
Other uses are in the works.
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-11 19:32 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
2024-05-15 00:39 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 15:50 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
@ 2024-05-18 16:50 ` Bruce Momjian <[email protected]>
2024-05-18 20:37 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-18 16:50 UTC (permalink / raw)
To: Andrew Dunstan <[email protected]>; +Cc: pgsql-hackers
On Thu, May 16, 2024 at 11:50:20AM -0400, Andrew Dunstan wrote:
> > Maybe "Introduce an incremental JSON parser" would have been a better
> > > headline.
> > Well, this gets into a level of detail that is beyond the average
> > reader. I think at that level people will need to read the git logs or
> > review the code. Do we use it for anything yet?
>
>
> Yes, certainly, it's used in handling backup manifests. Without it we can't
> handle huge manifests. See commits ea7b4e9a2a and 222e11a10a.
>
> Other uses are in the works.
Okay, added in the attached applied patch.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (659B, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 428cb5c5a2e..9c511848943 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -2451,6 +2451,17 @@ User-defined data type receive functions will no longer receive their data null-
</para>
</listitem>
+<!--
+Author: Andrew Dunstan <[email protected]>
+2024-04-04 [3311ea86e] Introduce a non-recursive JSON parser
+-->
+
+<listitem>
+<para>
+Add incremental JSON parser for use with huge JSON documents (Andrew Dunstan)
+</para>
+</listitem>
+
<!--
Author: Nathan Bossart <[email protected]>
2024-02-28 [363eb0599] Convert README to Markdown.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-11 19:32 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
2024-05-15 00:39 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 15:50 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
2024-05-18 16:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-18 20:37 ` Andrew Dunstan <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Andrew Dunstan @ 2024-05-18 20:37 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On 2024-05-18 Sa 12:50, Bruce Momjian wrote:
> On Thu, May 16, 2024 at 11:50:20AM -0400, Andrew Dunstan wrote:
>>> Maybe "Introduce an incremental JSON parser" would have been a better
>>>> headline.
>>> Well, this gets into a level of detail that is beyond the average
>>> reader. I think at that level people will need to read the git logs or
>>> review the code. Do we use it for anything yet?
>>
>> Yes, certainly, it's used in handling backup manifests. Without it we can't
>> handle huge manifests. See commits ea7b4e9a2a and 222e11a10a.
>>
>> Other uses are in the works.
> Okay, added in the attached applied patch.
>
Thanks
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-14 10:34 ` Elena Indrupskaya <[email protected]>
2024-05-15 00:43 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Elena Indrupskaya @ 2024-05-14 10:34 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; pgsql-hackers
Hi everybody,
Being a technical writer, I attached a small patch that fixes minor
language stuff.
Thank you.
Elena Indrupskaya
Postgres Professional Company
Moscow, Russia
On 09.05.2024 07:03, Bruce Momjian wrote:
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
> It will be improved until the final release. The item count is 188,
> which is similar to recent releases:
>
> release-10: 189
> release-11: 170
> release-12: 180
> release-13: 178
> release-14: 220
> release-15: 184
> release-16: 206
> release-17: 188
>
> I welcome feedback. For some reason it was an easier job than usual.
>
Attachments:
[text/x-patch] release_17.patch (2.2K, ../../[email protected]/2-release_17.patch)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 5a741ff16f..1ecf40fc73 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -73,7 +73,7 @@ Author: Michael Paquier <[email protected]>
<listitem>
<para>
-Restrict "ago" to only appear at the end of in interval values (Joseph Koshakow)
+Restrict "ago" to only appear at the end in interval values (Joseph Koshakow)
</para>
<para>
@@ -456,7 +456,7 @@ Author: Tomas Vondra <[email protected]>
<listitem>
<para>
-Allow a BRIN indexes to be created using parallel workers (Tomas Vondra, Matthias van de Meent)
+Allow BRIN indexes to be created using parallel workers (Tomas Vondra, Matthias van de Meent)
</para>
</listitem>
@@ -941,7 +941,7 @@ Allow the creation of WAL summarization files (Robert Haas, Nathan Bossart, Hube
</para>
<para>
-These files record the block numbers that have changed within an LSN range, and is useful for incremental file system backups. This is controlled by the server variables
+These files record the block numbers that have changed within an LSN range and are useful for incremental file system backups. This is controlled by the server variables
summarize_wal and wal_summarize_keep_time, and introspected with pg_available_wal_summaries(), pg_wal_summary_contents(), and pg_get_wal_summarizer_state().
</para>
</listitem>
@@ -1113,7 +1113,7 @@ Allow the application of logical replication changes to use hash indexes on the
</para>
<para>
-Previously only btree indexes could be use for this purpose.
+Previously only btree indexes could be used for this purpose.
</para>
</listitem>
@@ -2185,7 +2185,7 @@ Allow pg_basebackup and pg_receivewal to use dbname in their connection specific
</para>
<para>
-This is useful for connection poolers who are sensitive to the database name.
+This is useful for connection poolers that are sensitive to the database name.
</para>
</listitem>
@@ -2346,7 +2346,7 @@ Author: Michael Paquier <[email protected]>
<listitem>
<para>
-Remove the Microsoft Visual Studio Studio-specific Postgres build option (Michael Paquier)
+Remove the Microsoft Visual Studio-specific Postgres build option (Michael Paquier)
</para>
<para>
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 10:34 ` Re: First draft of PG 17 release notes Elena Indrupskaya <[email protected]>
@ 2024-05-15 00:43 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-15 00:43 UTC (permalink / raw)
To: Elena Indrupskaya <[email protected]>; +Cc: pgsql-hackers
On Tue, May 14, 2024 at 01:34:56PM +0300, Elena Indrupskaya wrote:
> Being a technical writer, I attached a small patch that fixes minor language
> stuff.
You are absolutely correct. Patch applied, thanks.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-14 13:58 ` Pantelis Theodosiou <[email protected]>
2024-05-15 00:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Pantelis Theodosiou @ 2024-05-14 13:58 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 5:03 AM Bruce Momjian <[email protected]> wrote
>
>
> I welcome feedback. For some reason it was an easier job than usual.
This looks better if "more case" -> "more cases" :
> Allow query nodes to be run in parallel in more case (Tom Lane)
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 13:58 ` Re: First draft of PG 17 release notes Pantelis Theodosiou <[email protected]>
@ 2024-05-15 00:48 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-15 00:48 UTC (permalink / raw)
To: Pantelis Theodosiou <[email protected]>; +Cc: pgsql-hackers
On Tue, May 14, 2024 at 02:58:41PM +0100, Pantelis Theodosiou wrote:
> On Thu, May 9, 2024 at 5:03 AM Bruce Momjian <[email protected]> wrote
> >
> >
> > I welcome feedback. For some reason it was an easier job than usual.
>
> This looks better if "more case" -> "more cases" :
> > Allow query nodes to be run in parallel in more case (Tom Lane)
Yes, you are correct, fixed.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-14 19:39 ` Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Melanie Plageman @ 2024-05-14 19:39 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 12:04 AM Bruce Momjian <[email protected]> wrote:
>
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
I had two comments:
--------
I think the read stream item:
"Allow the grouping of file system reads with the new system variable
io_combine_limit"
Might be better if it mentions the effect, like:
"Reduce system calls by automatically merging reads up to io_combine_limit"
-------
For the vacuum feature:
"Allow vacuum to more efficiently remove and freeze tuples"
I think that we need to more clearly point out the implications of the
feature added by Sawada-san (and reviewed by John) in 667e65aac35497.
Vacuum no longer uses a fixed amount of memory for dead tuple TID
storage and it is not preallocated. This affects users as they may
want to change their configuration (and expectations).
If you make that item more specific to their work, you should also
remove my name, as the work I did on vacuum this release was unrelated
to their work on dead tuple TID storage.
The work Heikki and I did which culminated in 6dbb490261 mainly has
the impact of improving vacuum's performance (vacuum emits less WAL
and is more efficient). So you could argue for removing it from the
release notes if you are using the requirement that performance
improvements don't go in the release notes.
However, one of the preliminary commits for this f83d70976 does change
WAL format. There are three WAL records which no longer exist as
separate records. Do users care about this?
- Melanie
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
@ 2024-05-15 01:00 ` Bruce Momjian <[email protected]>
2024-05-15 02:03 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
0 siblings, 2 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-15 01:00 UTC (permalink / raw)
To: Melanie Plageman <[email protected]>; +Cc: pgsql-hackers
On Tue, May 14, 2024 at 03:39:26PM -0400, Melanie Plageman wrote:
> On Thu, May 9, 2024 at 12:04 AM Bruce Momjian <[email protected]> wrote:
> >
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
>
> I had two comments:
>
> --------
> I think the read stream item:
>
> "Allow the grouping of file system reads with the new system variable
> io_combine_limit"
>
> Might be better if it mentions the effect, like:
>
> "Reduce system calls by automatically merging reads up to io_combine_limit"
Uh, as I understand it, the reduced number of system calls is not the
value of the feature, but rather the ability to request a larger block
from the I/O subsystem. Without it, you have to make a request and wait
for each request to finish. I am open to new wording, but I am not sure
your new wording is accurate.
> -------
> For the vacuum feature:
>
> "Allow vacuum to more efficiently remove and freeze tuples"
>
> I think that we need to more clearly point out the implications of the
> feature added by Sawada-san (and reviewed by John) in 667e65aac35497.
> Vacuum no longer uses a fixed amount of memory for dead tuple TID
> storage and it is not preallocated. This affects users as they may
> want to change their configuration (and expectations).
>
> If you make that item more specific to their work, you should also
> remove my name, as the work I did on vacuum this release was unrelated
> to their work on dead tuple TID storage.
>
> The work Heikki and I did which culminated in 6dbb490261 mainly has
> the impact of improving vacuum's performance (vacuum emits less WAL
> and is more efficient). So you could argue for removing it from the
> release notes if you are using the requirement that performance
> improvements don't go in the release notes.
>
> However, one of the preliminary commits for this f83d70976 does change
> WAL format. There are three WAL records which no longer exist as
> separate records. Do users care about this?
I don't think users really care about these details, just that it is
faster so they will not be surprised if there is a change. I was
purposely vague to group multiple commits into the single item. By
grouping them together, I got enough impact to warrant listing it. If
you split it apart, it is harder to justify mentioning them.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-15 02:03 ` David Rowley <[email protected]>
2024-05-15 02:06 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
1 sibling, 1 reply; 194+ messages in thread
From: David Rowley @ 2024-05-15 02:03 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Melanie Plageman <[email protected]>; pgsql-hackers
On Wed, 15 May 2024 at 13:00, Bruce Momjian <[email protected]> wrote:
>
> On Tue, May 14, 2024 at 03:39:26PM -0400, Melanie Plageman wrote:
> > "Reduce system calls by automatically merging reads up to io_combine_limit"
>
> Uh, as I understand it, the reduced number of system calls is not the
> value of the feature, but rather the ability to request a larger block
> from the I/O subsystem. Without it, you have to make a request and wait
> for each request to finish. I am open to new wording, but I am not sure
> your new wording is accurate.
I think you have the cause and effect backwards. There's no advantage
to reading 128KB if you only need 8KB. It's the fact that doing
*larger* reads allows *fewer* reads that allows it to be more
efficient. There are also the efficiency gains from fadvise
POSIX_FADV_WILLNEED. I'm unsure how to jam that into a short sentence.
Maybe; "Optimize reading of tables by allowing pages to be prefetched
and read in chunks up to io_combine_limit", or a bit more buzzy;
"Optimize reading of tables by allowing pages to be prefetched and
performing vectored reads in chunks up to io_combine_limit".
David
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 02:03 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
@ 2024-05-15 02:06 ` Bruce Momjian <[email protected]>
2024-05-15 02:24 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Amit Kapila <[email protected]>
0 siblings, 2 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-15 02:06 UTC (permalink / raw)
To: David Rowley <[email protected]>; +Cc: Melanie Plageman <[email protected]>; pgsql-hackers
On Wed, May 15, 2024 at 02:03:32PM +1200, David Rowley wrote:
> On Wed, 15 May 2024 at 13:00, Bruce Momjian <[email protected]> wrote:
> >
> > On Tue, May 14, 2024 at 03:39:26PM -0400, Melanie Plageman wrote:
> > > "Reduce system calls by automatically merging reads up to io_combine_limit"
> >
> > Uh, as I understand it, the reduced number of system calls is not the
> > value of the feature, but rather the ability to request a larger block
> > from the I/O subsystem. Without it, you have to make a request and wait
> > for each request to finish. I am open to new wording, but I am not sure
> > your new wording is accurate.
>
> I think you have the cause and effect backwards. There's no advantage
> to reading 128KB if you only need 8KB. It's the fact that doing
> *larger* reads allows *fewer* reads that allows it to be more
> efficient. There are also the efficiency gains from fadvise
> POSIX_FADV_WILLNEED. I'm unsure how to jam that into a short sentence.
> Maybe; "Optimize reading of tables by allowing pages to be prefetched
> and read in chunks up to io_combine_limit", or a bit more buzzy;
> "Optimize reading of tables by allowing pages to be prefetched and
> performing vectored reads in chunks up to io_combine_limit".
Yes, my point is that it is not the number of system calls or system
call overhead that is the advantage of this patch, but the ability to
request more of the I/O system in one call, which is not the same as
system calls.
I can use your wording, but how much prefetching to we enable now?
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 02:03 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-15 02:06 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-15 02:24 ` David Rowley <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: David Rowley @ 2024-05-15 02:24 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Melanie Plageman <[email protected]>; pgsql-hackers
On Wed, 15 May 2024 at 14:06, Bruce Momjian <[email protected]> wrote:
> I can use your wording, but how much prefetching to we enable now?
I believe the read stream API is used for Seq Scan, ANALYZE and
pg_prewarm(). fadvise() is used when the next buffer that's required
is not in shared buffers on any build that has defined
HAVE_DECL_POSIX_FADVISE.
David
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 02:03 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-15 02:06 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-15 13:13 ` Amit Kapila <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Amit Kapila @ 2024-05-15 13:13 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: David Rowley <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Wed, May 15, 2024 at 7:36 AM Bruce Momjian <[email protected]> wrote:
>
> On Wed, May 15, 2024 at 02:03:32PM +1200, David Rowley wrote:
> > On Wed, 15 May 2024 at 13:00, Bruce Momjian <[email protected]> wrote:
> > >
> > > On Tue, May 14, 2024 at 03:39:26PM -0400, Melanie Plageman wrote:
> > > > "Reduce system calls by automatically merging reads up to io_combine_limit"
> > >
> > > Uh, as I understand it, the reduced number of system calls is not the
> > > value of the feature, but rather the ability to request a larger block
> > > from the I/O subsystem. Without it, you have to make a request and wait
> > > for each request to finish. I am open to new wording, but I am not sure
> > > your new wording is accurate.
> >
> > I think you have the cause and effect backwards. There's no advantage
> > to reading 128KB if you only need 8KB. It's the fact that doing
> > *larger* reads allows *fewer* reads that allows it to be more
> > efficient. There are also the efficiency gains from fadvise
> > POSIX_FADV_WILLNEED. I'm unsure how to jam that into a short sentence.
> > Maybe; "Optimize reading of tables by allowing pages to be prefetched
> > and read in chunks up to io_combine_limit", or a bit more buzzy;
> > "Optimize reading of tables by allowing pages to be prefetched and
> > performing vectored reads in chunks up to io_combine_limit".
>
> Yes, my point is that it is not the number of system calls or system
> call overhead that is the advantage of this patch, but the ability to
> request more of the I/O system in one call, which is not the same as
> system calls.
>
> I can use your wording, but how much prefetching to we enable now?
>
Shouldn't we need to include commit
b5a9b18cd0bc6f0124664999b31a00a264d16913 with this item?
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-15 08:38 ` Alvaro Herrera <[email protected]>
2024-05-15 11:17 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
1 sibling, 3 replies; 194+ messages in thread
From: Alvaro Herrera @ 2024-05-15 08:38 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Melanie Plageman <[email protected]>; pgsql-hackers
On 2024-May-14, Bruce Momjian wrote:
> On Tue, May 14, 2024 at 03:39:26PM -0400, Melanie Plageman wrote:
> > I think that we need to more clearly point out the implications of the
> > feature added by Sawada-san (and reviewed by John) in 667e65aac35497.
> > Vacuum no longer uses a fixed amount of memory for dead tuple TID
> > storage and it is not preallocated. This affects users as they may
> > want to change their configuration (and expectations).
> >
> > If you make that item more specific to their work, you should also
> > remove my name, as the work I did on vacuum this release was unrelated
> > to their work on dead tuple TID storage.
> >
> > The work Heikki and I did which culminated in 6dbb490261 mainly has
> > the impact of improving vacuum's performance (vacuum emits less WAL
> > and is more efficient). So you could argue for removing it from the
> > release notes if you are using the requirement that performance
> > improvements don't go in the release notes.
>
> I don't think users really care about these details, just that it is
> faster so they will not be surprised if there is a change. I was
> purposely vague to group multiple commits into the single item. By
> grouping them together, I got enough impact to warrant listing it. If
> you split it apart, it is harder to justify mentioning them.
I disagree with this. IMO the impact of the Sawada/Naylor change is
likely to be enormous for people with large tables and large numbers of
tuples to clean up (I know we've had a number of customers in this
situation, I can't imagine any Postgres service provider that doesn't).
The fact that maintenance_work_mem is no longer capped at 1GB is very
important and I think we should mention that explicitly in the release
notes, as setting it higher could make a big difference in vacuum run
times.
I don't know what's the impact of the Plageman/Linnakangas work, but
since there are no user-visible consequences other than it being faster,
I agree it could be put more succintly, perhaps together as a sub-para
of the same item.
What about something like this?
<para>
Lift the 1 GB allocation limit for vacuum memory usage for dead
tuples, and make storage more compact and performant.
</para>
<para>
This can reduce the number of index passes that vacuum has to perform
for tables with many dead tuples, shortening vacuum times.
</para>
<para>
Also, the WAL traffic caused by vacuum has been made more compact.
</para>
> > However, one of the preliminary commits for this f83d70976 does
> > change WAL format. There are three WAL records which no longer exist
> > as separate records. Do users care about this?
I don't think so.
--
Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/
"You don't solve a bad join with SELECT DISTINCT" #CupsOfFail
https://twitter.com/connor_mc_d/status/1431240081726115845
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
@ 2024-05-15 11:17 ` David Rowley <[email protected]>
2 siblings, 0 replies; 194+ messages in thread
From: David Rowley @ 2024-05-15 11:17 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Melanie Plageman <[email protected]>; pgsql-hackers; Alvaro Herrera <[email protected]>
On Wed, 15 May 2024 at 20:38, Alvaro Herrera <[email protected]> wrote:
>
> On 2024-May-14, Bruce Momjian wrote:
> > I don't think users really care about these details, just that it is
> > faster so they will not be surprised if there is a change. I was
> > purposely vague to group multiple commits into the single item. By
> > grouping them together, I got enough impact to warrant listing it. If
> > you split it apart, it is harder to justify mentioning them.
>
> I disagree with this. IMO the impact of the Sawada/Naylor change is
> likely to be enormous for people with large tables and large numbers of
> tuples to clean up (I know we've had a number of customers in this
> situation, I can't imagine any Postgres service provider that doesn't).
> The fact that maintenance_work_mem is no longer capped at 1GB is very
> important and I think we should mention that explicitly in the release
> notes, as setting it higher could make a big difference in vacuum run
> times.
I very much agree with Alvaro here. IMO, this should be on the
highlight feature list for v17. Prior to this, having to do multiple
index scans because of filling maintenance_work_mem was a performance
tragedy. If there were enough dead tuples to have filled
maintenance_work_mem, then the indexes are large. Having to scan
multiple large indexes multiple times isn't good use of I/O and CPU.
As far as I understand it, this work means it'll be unlikely that a
well-configured server will ever have to do multiple index passes. I
don't think "enormous impact" is an exaggeration here.
David
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
@ 2024-05-15 13:13 ` Melanie Plageman <[email protected]>
2024-05-16 02:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2 siblings, 1 reply; 194+ messages in thread
From: Melanie Plageman @ 2024-05-15 13:13 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Bruce Momjian <[email protected]>; pgsql-hackers
On Wed, May 15, 2024 at 4:38 AM Alvaro Herrera <[email protected]> wrote:
>
> On 2024-May-14, Bruce Momjian wrote:
>
> > On Tue, May 14, 2024 at 03:39:26PM -0400, Melanie Plageman wrote:
>
> > > I think that we need to more clearly point out the implications of the
> > > feature added by Sawada-san (and reviewed by John) in 667e65aac35497.
> > > Vacuum no longer uses a fixed amount of memory for dead tuple TID
> > > storage and it is not preallocated. This affects users as they may
> > > want to change their configuration (and expectations).
> > >
> > > If you make that item more specific to their work, you should also
> > > remove my name, as the work I did on vacuum this release was unrelated
> > > to their work on dead tuple TID storage.
> > >
> > > The work Heikki and I did which culminated in 6dbb490261 mainly has
> > > the impact of improving vacuum's performance (vacuum emits less WAL
> > > and is more efficient). So you could argue for removing it from the
> > > release notes if you are using the requirement that performance
> > > improvements don't go in the release notes.
> >
> > I don't think users really care about these details, just that it is
> > faster so they will not be surprised if there is a change. I was
> > purposely vague to group multiple commits into the single item. By
> > grouping them together, I got enough impact to warrant listing it. If
> > you split it apart, it is harder to justify mentioning them.
>
> I disagree with this. IMO the impact of the Sawada/Naylor change is
> likely to be enormous for people with large tables and large numbers of
> tuples to clean up (I know we've had a number of customers in this
> situation, I can't imagine any Postgres service provider that doesn't).
> The fact that maintenance_work_mem is no longer capped at 1GB is very
> important and I think we should mention that explicitly in the release
> notes, as setting it higher could make a big difference in vacuum run
> times.
>
> I don't know what's the impact of the Plageman/Linnakangas work, but
> since there are no user-visible consequences other than it being faster,
> I agree it could be put more succintly, perhaps together as a sub-para
> of the same item.
>
> What about something like this?
>
> <para>
> Lift the 1 GB allocation limit for vacuum memory usage for dead
> tuples, and make storage more compact and performant.
> </para>
> <para>
> This can reduce the number of index passes that vacuum has to perform
> for tables with many dead tuples, shortening vacuum times.
> </para>
> <para>
> Also, the WAL traffic caused by vacuum has been made more compact.
> </para>
I think this wording and organization makes sense. I hadn't thought of
using "traffic" to describe this, but I like it.
Also +1 on the Sawada/Naylor change being on the highlight section of
the release (as David suggested upthread).
- Melanie
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
@ 2024-05-16 02:48 ` Bruce Momjian <[email protected]>
2024-05-16 03:35 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-16 02:48 UTC (permalink / raw)
To: Melanie Plageman <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; pgsql-hackers
On Wed, May 15, 2024 at 09:13:14AM -0400, Melanie Plageman wrote:
> I think this wording and organization makes sense. I hadn't thought of
> using "traffic" to describe this, but I like it.
>
> Also +1 on the Sawada/Naylor change being on the highlight section of
> the release (as David suggested upthread).
Agreed, I went with the attached applied patch.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (1.2K, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index e68c499e0db..7f36a35954d 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -519,6 +519,25 @@ Allow vacuum to more efficiently remove and freeze tuples (Masahiko Sawada, John
</para>
</listitem>
+<!--
+Author: Masahiko Sawada <[email protected]>
+2024-03-21 [30e144287] Add TIDStore, to store sets of TIDs (ItemPointerData) ef
+Author: Masahiko Sawada <[email protected]>
+2024-04-02 [667e65aac] Use TidStore for dead tuple TIDs storage during lazy vac
+Author: Heikki Linnakangas <[email protected]>
+2024-04-03 [6dbb49026] Combine freezing and pruning steps in VACUUM
+-->
+
+<listitem>
+<para>
+Allow vacuum to more efficiently store tuple references and remove its memory limit (Masahiko Sawada, John Naylor)
+</para>
+
+<para>
+Specifically, maintenance_work_mem and autovacuum_work_mem can now be configured to use more than one gigabyte of memory. WAL traffic caused by vacuum is also more compact.
+</para>
+</listitem>
+
<!--
Author: Robert Haas <[email protected]>
2024-01-18 [c120550ed] Optimize vacuuming of relations with no indexes.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-16 02:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-16 03:35 ` David Rowley <[email protected]>
2024-05-18 14:40 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: David Rowley @ 2024-05-16 03:35 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Melanie Plageman <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Thu, 16 May 2024 at 14:48, Bruce Momjian <[email protected]> wrote:
>
> On Wed, May 15, 2024 at 09:13:14AM -0400, Melanie Plageman wrote:
> > Also +1 on the Sawada/Naylor change being on the highlight section of
> > the release (as David suggested upthread).
>
> Agreed, I went with the attached applied patch.
+Allow vacuum to more efficiently store tuple references and remove
its memory limit (Masahiko Sawada, John Naylor)
+</para>
I don't want it to seem like I'm splitting hairs, but I'd drop the "
and remove its memory limit"
+<para>
+Specifically, maintenance_work_mem and autovacuum_work_mem can now be
configured to use more than one gigabyte of memory. WAL traffic
caused by vacuum is also more compact.
I'd say the first sentence above should be written as:
"Additionally, vacuum no longer silently imposes a 1GB tuple reference
limit even when maintenance_work_mem or autovacuum_work_mem are set to
higher values"
It's not "Specifically" as the "more efficiently store tuple
references" isn't the same thing as removing the 1GB cap. Also, there
was never a restriction in configuring maintenance_work_mem or
autovacuum_work_mem to values higher than 1GB. The restriction was
that vacuum was unable to utilize anything more than that.
David
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-16 02:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 03:35 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
@ 2024-05-18 14:40 ` Bruce Momjian <[email protected]>
2024-05-19 03:53 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-18 14:40 UTC (permalink / raw)
To: David Rowley <[email protected]>; +Cc: Melanie Plageman <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Thu, May 16, 2024 at 03:35:17PM +1200, David Rowley wrote:
> On Thu, 16 May 2024 at 14:48, Bruce Momjian <[email protected]> wrote:
> >
> > On Wed, May 15, 2024 at 09:13:14AM -0400, Melanie Plageman wrote:
> > > Also +1 on the Sawada/Naylor change being on the highlight section of
> > > the release (as David suggested upthread).
> >
> > Agreed, I went with the attached applied patch.
>
> +Allow vacuum to more efficiently store tuple references and remove
> its memory limit (Masahiko Sawada, John Naylor)
> +</para>
>
> I don't want it to seem like I'm splitting hairs, but I'd drop the "
> and remove its memory limit"
>
> +<para>
> +Specifically, maintenance_work_mem and autovacuum_work_mem can now be
> configured to use more than one gigabyte of memory. WAL traffic
> caused by vacuum is also more compact.
>
> I'd say the first sentence above should be written as:
>
> "Additionally, vacuum no longer silently imposes a 1GB tuple reference
> limit even when maintenance_work_mem or autovacuum_work_mem are set to
> higher values"
>
> It's not "Specifically" as the "more efficiently store tuple
> references" isn't the same thing as removing the 1GB cap. Also, there
> was never a restriction in configuring maintenance_work_mem or
> autovacuum_work_mem to values higher than 1GB. The restriction was
> that vacuum was unable to utilize anything more than that.
Slightly adjusted wording patch attached and applied.
My deep apologies for the delay in addressing this. I should have done
it sooner.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (894B, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index ed1af17bb0a..d7fd87f3d5c 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -530,11 +530,12 @@ Author: Heikki Linnakangas <[email protected]>
<listitem>
<para>
-Allow vacuum to more efficiently store tuple references and remove its memory limit (Masahiko Sawada, John Naylor)
+Allow vacuum to more efficiently store tuple references (Masahiko Sawada, John Naylor)
</para>
<para>
-Specifically, maintenance_work_mem and autovacuum_work_mem can now be configured to use more than one gigabyte of memory. WAL traffic caused by vacuum is also more compact.
+Additionally, vacuum is no longer silently limited to one gigabyte of memory when maintenance_work_mem and autovacuum_work_mem are higher. WAL traffic caused by
+vacuum is also more compact.
</para>
</listitem>
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-16 02:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 03:35 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-18 14:40 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-19 03:53 ` David Rowley <[email protected]>
2024-05-20 00:12 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: David Rowley @ 2024-05-19 03:53 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Melanie Plageman <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Sun, 19 May 2024 at 02:40, Bruce Momjian <[email protected]> wrote:
>
> On Thu, May 16, 2024 at 03:35:17PM +1200, David Rowley wrote:
> > "Additionally, vacuum no longer silently imposes a 1GB tuple reference
> > limit even when maintenance_work_mem or autovacuum_work_mem are set to
> > higher values"
> Slightly adjusted wording patch attached and applied.
Thanks for adjusting.
It's a minor detail, but I'll mention it because you went to the
effort to adjust it away from what I'd written...
I didn't make a random choice to use "or" between the two GUCs.
Changing it to "and", IMO, isn't an improvement. Using "and" implies
that the silent limited was only imposed when both of these GUCs were
set >= 1GB. That's not true. For the case we're talking about here, if
autovacuum_work_mem is set to anything apart from -1 then the value of
maintenance_work_mem does not matter.
David
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-16 02:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 03:35 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-18 14:40 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-19 03:53 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
@ 2024-05-20 00:12 ` Bruce Momjian <[email protected]>
2024-05-20 06:23 ` Re: First draft of PG 17 release notes John Naylor <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-20 00:12 UTC (permalink / raw)
To: David Rowley <[email protected]>; +Cc: Melanie Plageman <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Sun, May 19, 2024 at 03:53:38PM +1200, David Rowley wrote:
> On Sun, 19 May 2024 at 02:40, Bruce Momjian <[email protected]> wrote:
> >
> > On Thu, May 16, 2024 at 03:35:17PM +1200, David Rowley wrote:
> > > "Additionally, vacuum no longer silently imposes a 1GB tuple reference
> > > limit even when maintenance_work_mem or autovacuum_work_mem are set to
> > > higher values"
>
> > Slightly adjusted wording patch attached and applied.
>
> Thanks for adjusting.
>
> It's a minor detail, but I'll mention it because you went to the
> effort to adjust it away from what I'd written...
True.
> I didn't make a random choice to use "or" between the two GUCs.
> Changing it to "and", IMO, isn't an improvement. Using "and" implies
> that the silent limited was only imposed when both of these GUCs were
> set >= 1GB. That's not true. For the case we're talking about here, if
> autovacuum_work_mem is set to anything apart from -1 then the value of
> maintenance_work_mem does not matter.
Okay, changed to "or".
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-16 02:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 03:35 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-18 14:40 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-19 03:53 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-20 00:12 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-20 06:23 ` John Naylor <[email protected]>
2024-05-20 13:37 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: John Naylor @ 2024-05-20 06:23 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: David Rowley <[email protected]>; Melanie Plageman <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
Hi Bruce, thanks for doing this again!
I'm a bit late to this discussion -- there's been a bit of churn in
the vacuum items, and some streams got crossed along the way. I've
attached an attempt to rectify this.
Attachments:
[text/x-patch] adjust-vacuum-credits.patch (1.8K, ../../CANWCAZbPEoFcSjedGsz4AV=_sbwUN4sgjrukVCu-JzrqFC5Hhw@mail.gmail.com/2-adjust-vacuum-credits.patch)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 0e7ff51f4a..612eb100a7 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -492,23 +492,23 @@ Allow BRIN indexes to be created using parallel workers (Tomas Vondra, Matthias
<itemizedlist>
<!--
-Author: John Naylor <[email protected]>
-2024-03-07 [ee1b30f12] Add template for adaptive radix tree
-Author: Masahiko Sawada <[email protected]>
-2024-03-21 [30e144287] Add TIDStore, to store sets of TIDs (ItemPointerData) ef
-Author: Masahiko Sawada <[email protected]>
-2024-04-02 [667e65aac] Use TidStore for dead tuple TIDs storage during lazy vac
Author: Heikki Linnakangas <[email protected]>
2024-04-03 [6dbb49026] Combine freezing and pruning steps in VACUUM
-->
<listitem>
<para>
-Allow vacuum to more efficiently remove and freeze tuples (Masahiko Sawada, John Naylor, Melanie Plageman)
+Allow vacuum to more efficiently remove and freeze tuples (Melanie Plageman)
+</para>
+
+<para>
+WAL traffic caused by vacuum is also more compact.
</para>
</listitem>
<!--
+Author: John Naylor <[email protected]>
+2024-03-07 [ee1b30f12] Add template for adaptive radix tree
Author: Masahiko Sawada <[email protected]>
2024-03-21 [30e144287] Add TIDStore, to store sets of TIDs (ItemPointerData) ef
Author: Masahiko Sawada <[email protected]>
@@ -523,8 +523,7 @@ Allow vacuum to more efficiently store tuple references (Masahiko Sawada, John N
</para>
<para>
-Additionally, vacuum is no longer silently limited to one gigabyte of memory when maintenance_work_mem or autovacuum_work_mem are higher. WAL traffic caused by
-vacuum is also more compact.
+Additionally, vacuum is no longer silently limited to one gigabyte of memory when maintenance_work_mem or autovacuum_work_mem are higher.
</para>
</listitem>
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-16 02:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 03:35 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-18 14:40 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-19 03:53 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-20 00:12 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 06:23 ` Re: First draft of PG 17 release notes John Naylor <[email protected]>
@ 2024-05-20 13:37 ` Bruce Momjian <[email protected]>
2024-05-20 18:47 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-20 13:37 UTC (permalink / raw)
To: John Naylor <[email protected]>; +Cc: David Rowley <[email protected]>; Melanie Plageman <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Mon, May 20, 2024 at 01:23:02PM +0700, John Naylor wrote:
> Hi Bruce, thanks for doing this again!
>
> I'm a bit late to this discussion -- there's been a bit of churn in
> the vacuum items, and some streams got crossed along the way. I've
> attached an attempt to rectify this.
Agreed, patch applied, thanks.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-16 02:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 03:35 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-18 14:40 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-19 03:53 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-20 00:12 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 06:23 ` Re: First draft of PG 17 release notes John Naylor <[email protected]>
2024-05-20 13:37 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-20 18:47 ` Melanie Plageman <[email protected]>
2024-05-22 21:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Melanie Plageman @ 2024-05-20 18:47 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: John Naylor <[email protected]>; David Rowley <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Mon, May 20, 2024 at 9:37 AM Bruce Momjian <[email protected]> wrote:
>
> On Mon, May 20, 2024 at 01:23:02PM +0700, John Naylor wrote:
> > Hi Bruce, thanks for doing this again!
> >
> > I'm a bit late to this discussion -- there's been a bit of churn in
> > the vacuum items, and some streams got crossed along the way. I've
> > attached an attempt to rectify this.
>
> Agreed, patch applied, thanks.
If "Allow vacuum to more efficiently remove and freeze tuples" stays
in the release notes, I would add Heikki's name. He wasn't listed as a
co-author on all of the commits that were part of this feature, but he
made a substantial investment in it and should be listed, IMO. Patch
attached.
- Melanie
Attachments:
[text/x-patch] add_heikki.patch (829B, ../../CAAKRu_YYR5MXY-xuCpr7DKKugCExTSjRy9Ax8c-z7LKa8dDfwg@mail.gmail.com/2-add_heikki.patch)
download | inline diff:
From 6f8c775a45ef90a3882d3d2f3c484fc82f9c072a Mon Sep 17 00:00:00 2001
From: Melanie Plageman <[email protected]>
Date: Mon, 20 May 2024 14:44:41 -0400
Subject: [PATCH v1] doc PG 17 relnotes: add vacuum item author
Reported-by: Melanie Plageman
---
doc/src/sgml/release-17.sgml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 612eb100a7b..3025f31f8a0 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -498,7 +498,7 @@ Author: Heikki Linnakangas <[email protected]>
<listitem>
<para>
-Allow vacuum to more efficiently remove and freeze tuples (Melanie Plageman)
+Allow vacuum to more efficiently remove and freeze tuples (Melanie Plageman, Heikki Linnakangas)
</para>
<para>
--
2.34.1
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-16 02:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 03:35 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-18 14:40 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-19 03:53 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-20 00:12 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 06:23 ` Re: First draft of PG 17 release notes John Naylor <[email protected]>
2024-05-20 13:37 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 18:47 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
@ 2024-05-22 21:59 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-22 21:59 UTC (permalink / raw)
To: Melanie Plageman <[email protected]>; +Cc: John Naylor <[email protected]>; David Rowley <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Mon, May 20, 2024 at 02:47:28PM -0400, Melanie Plageman wrote:
> On Mon, May 20, 2024 at 9:37 AM Bruce Momjian <[email protected]> wrote:
> >
> > On Mon, May 20, 2024 at 01:23:02PM +0700, John Naylor wrote:
> > > Hi Bruce, thanks for doing this again!
> > >
> > > I'm a bit late to this discussion -- there's been a bit of churn in
> > > the vacuum items, and some streams got crossed along the way. I've
> > > attached an attempt to rectify this.
> >
> > Agreed, patch applied, thanks.
>
> If "Allow vacuum to more efficiently remove and freeze tuples" stays
> in the release notes, I would add Heikki's name. He wasn't listed as a
> co-author on all of the commits that were part of this feature, but he
> made a substantial investment in it and should be listed, IMO. Patch
> attached.
Thanks, patch applied.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
@ 2024-05-16 03:48 ` Andres Freund <[email protected]>
2024-05-16 09:49 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-16 12:09 ` Re: First draft of PG 17 release notes Joe Conway <[email protected]>
2024-05-16 13:09 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-16 14:55 ` Re: First draft of PG 17 release notes Peter Geoghegan <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2 siblings, 5 replies; 194+ messages in thread
From: Andres Freund @ 2024-05-16 03:48 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Bruce Momjian <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
Hi,
On 2024-05-15 10:38:20 +0200, Alvaro Herrera wrote:
> I disagree with this. IMO the impact of the Sawada/Naylor change is
> likely to be enormous for people with large tables and large numbers of
> tuples to clean up (I know we've had a number of customers in this
> situation, I can't imagine any Postgres service provider that doesn't).
> The fact that maintenance_work_mem is no longer capped at 1GB is very
> important and I think we should mention that explicitly in the release
> notes, as setting it higher could make a big difference in vacuum run
> times.
+many.
We're having this debate every release. I think the ongoing reticence to note
performance improvements in the release notes is hurting Postgres.
For one, performance improvements are one of the prime reason users
upgrade. Without them being noted anywhere more dense than the commit log,
it's very hard to figure out what improved for users. A halfway widely
applicable performance improvement is far more impactful than many of the
feature changes we do list in the release notes.
For another, it's also very frustrating for developers that focus on
performance. The reticence to note their work, while noting other, far
smaller, things in the release notes, pretty much tells us that our work isn't
valued.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-16 09:49 ` Jelte Fennema-Nio <[email protected]>
4 siblings, 0 replies; 194+ messages in thread
From: Jelte Fennema-Nio @ 2024-05-16 09:49 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Bruce Momjian <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Thu, 16 May 2024 at 05:48, Andres Freund <[email protected]> wrote:
> We're having this debate every release. I think the ongoing reticence to note
> performance improvements in the release notes is hurting Postgres.
>
> For one, performance improvements are one of the prime reason users
> upgrade. Without them being noted anywhere more dense than the commit log,
> it's very hard to figure out what improved for users. A halfway widely
> applicable performance improvement is far more impactful than many of the
> feature changes we do list in the release notes.
>
> For another, it's also very frustrating for developers that focus on
> performance. The reticence to note their work, while noting other, far
> smaller, things in the release notes, pretty much tells us that our work isn't
> valued.
+1 to the general gist of listing every perf improvement **and memory
usage reduction** in the release notes. Most of them are already
grouped together in a dedicated "General performance" section anyway,
having that section be big would only be good imho to show that we're
committed to improving perf.
I think one thing would make this a lot easier though is if commits
that knowlingy impact perf would clearly say so in the commit message,
because now it's sometimes hard to spot as someone not deeply involved
with the specific patch. e.g. c4ab7da606 doesn't mention performance
at all, so I'm not surprised it wasn't listed initially. And while
667e65aac3 states that multiple rounds of heap scanning is now
extremely rare, it doesn't explicitly state what the kind of perf
impact can be expected because of that.
Maybe something like introducing a common "Perf-Improvement: true"
marker in the commit message and when doing so add a clear paragraph
explaining the expected perf impact perf impact. Another option could
be to add a "User Impact" section to the commit message, where an
author could add their suggestion for a release note entry. So
basically this suggestion boils down to more clearly mentioning user
impact in commit messages, instead of mostly/only including
technical/implementation details.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-16 12:09 ` Joe Conway <[email protected]>
4 siblings, 0 replies; 194+ messages in thread
From: Joe Conway @ 2024-05-16 12:09 UTC (permalink / raw)
To: Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; +Cc: Bruce Momjian <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On 5/15/24 23:48, Andres Freund wrote:
> On 2024-05-15 10:38:20 +0200, Alvaro Herrera wrote:
>> I disagree with this. IMO the impact of the Sawada/Naylor change is
>> likely to be enormous for people with large tables and large numbers of
>> tuples to clean up (I know we've had a number of customers in this
>> situation, I can't imagine any Postgres service provider that doesn't).
>> The fact that maintenance_work_mem is no longer capped at 1GB is very
>> important and I think we should mention that explicitly in the release
>> notes, as setting it higher could make a big difference in vacuum run
>> times.
>
> +many.
>
> We're having this debate every release. I think the ongoing reticence to note
> performance improvements in the release notes is hurting Postgres.
>
> For one, performance improvements are one of the prime reason users
> upgrade. Without them being noted anywhere more dense than the commit log,
> it's very hard to figure out what improved for users. A halfway widely
> applicable performance improvement is far more impactful than many of the
> feature changes we do list in the release notes.
many++
> For another, it's also very frustrating for developers that focus on
> performance. The reticence to note their work, while noting other, far
> smaller, things in the release notes, pretty much tells us that our work isn't
> valued.
agreed
--
Joe Conway
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-16 13:09 ` Melanie Plageman <[email protected]>
2024-05-18 15:13 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
4 siblings, 1 reply; 194+ messages in thread
From: Melanie Plageman @ 2024-05-16 13:09 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers
On Wed, May 15, 2024 at 11:48 PM Andres Freund <[email protected]> wrote:
>
> On 2024-05-15 10:38:20 +0200, Alvaro Herrera wrote:
> > I disagree with this. IMO the impact of the Sawada/Naylor change is
> > likely to be enormous for people with large tables and large numbers of
> > tuples to clean up (I know we've had a number of customers in this
> > situation, I can't imagine any Postgres service provider that doesn't).
> > The fact that maintenance_work_mem is no longer capped at 1GB is very
> > important and I think we should mention that explicitly in the release
> > notes, as setting it higher could make a big difference in vacuum run
> > times.
>
> +many.
>
> We're having this debate every release. I think the ongoing reticence to note
> performance improvements in the release notes is hurting Postgres.
>
> For one, performance improvements are one of the prime reason users
> upgrade. Without them being noted anywhere more dense than the commit log,
> it's very hard to figure out what improved for users. A halfway widely
> applicable performance improvement is far more impactful than many of the
> feature changes we do list in the release notes.
The practical reason this matters to users is that they want to change
their configuration or expectations in response to performance
improvements.
And also, as Jelte mentions upthread, describing performance
improvements made each release in Postgres makes it clear that we are
consistently improving it.
> For another, it's also very frustrating for developers that focus on
> performance. The reticence to note their work, while noting other, far
> smaller, things in the release notes, pretty much tells us that our work isn't
> valued.
+many
- Melanie
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-16 13:09 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
@ 2024-05-18 15:13 ` Bruce Momjian <[email protected]>
2024-05-20 18:35 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-21 16:40 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
0 siblings, 2 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-18 15:13 UTC (permalink / raw)
To: Melanie Plageman <[email protected]>; +Cc: Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Thu, May 16, 2024 at 09:09:11AM -0400, Melanie Plageman wrote:
> On Wed, May 15, 2024 at 11:48 PM Andres Freund <[email protected]> wrote:
> >
> > On 2024-05-15 10:38:20 +0200, Alvaro Herrera wrote:
> > > I disagree with this. IMO the impact of the Sawada/Naylor change is
> > > likely to be enormous for people with large tables and large numbers of
> > > tuples to clean up (I know we've had a number of customers in this
> > > situation, I can't imagine any Postgres service provider that doesn't).
> > > The fact that maintenance_work_mem is no longer capped at 1GB is very
> > > important and I think we should mention that explicitly in the release
> > > notes, as setting it higher could make a big difference in vacuum run
> > > times.
> >
> > +many.
> >
> > We're having this debate every release. I think the ongoing reticence to note
> > performance improvements in the release notes is hurting Postgres.
> >
> > For one, performance improvements are one of the prime reason users
> > upgrade. Without them being noted anywhere more dense than the commit log,
> > it's very hard to figure out what improved for users. A halfway widely
> > applicable performance improvement is far more impactful than many of the
> > feature changes we do list in the release notes.
>
> The practical reason this matters to users is that they want to change
> their configuration or expectations in response to performance
> improvements.
>
> And also, as Jelte mentions upthread, describing performance
> improvements made each release in Postgres makes it clear that we are
> consistently improving it.
>
> > For another, it's also very frustrating for developers that focus on
> > performance. The reticence to note their work, while noting other, far
> > smaller, things in the release notes, pretty much tells us that our work isn't
> > valued.
>
> +many
Please see the email I just posted. There are three goals we have to
adjust for:
1. short release notes so they are readable
2. giving people credit for performance improvements
3. showing people Postgres cares about performance
I would like to achieve 2 & 3 without harming #1. My experience is if I
am reading a long document, and I get to a section where I start to
wonder, "Why should I care about this?", I start to skim the rest of
the document. I am particularly critical if I start to wonder, "Why
does the author _think_ I should care about this?" becasue it feels like
the author is writing for him/herself and not the audience.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-16 13:09 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-18 15:13 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-20 18:35 ` Melanie Plageman <[email protected]>
2024-05-20 18:40 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
1 sibling, 1 reply; 194+ messages in thread
From: Melanie Plageman @ 2024-05-20 18:35 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Sat, May 18, 2024 at 11:13 AM Bruce Momjian <[email protected]> wrote:
>
> On Thu, May 16, 2024 at 09:09:11AM -0400, Melanie Plageman wrote:
> > On Wed, May 15, 2024 at 11:48 PM Andres Freund <[email protected]> wrote:
> > >
> > > On 2024-05-15 10:38:20 +0200, Alvaro Herrera wrote:
> > > > I disagree with this. IMO the impact of the Sawada/Naylor change is
> > > > likely to be enormous for people with large tables and large numbers of
> > > > tuples to clean up (I know we've had a number of customers in this
> > > > situation, I can't imagine any Postgres service provider that doesn't).
> > > > The fact that maintenance_work_mem is no longer capped at 1GB is very
> > > > important and I think we should mention that explicitly in the release
> > > > notes, as setting it higher could make a big difference in vacuum run
> > > > times.
> > >
> > > +many.
> > >
> > > We're having this debate every release. I think the ongoing reticence to note
> > > performance improvements in the release notes is hurting Postgres.
> > >
> > > For one, performance improvements are one of the prime reason users
> > > upgrade. Without them being noted anywhere more dense than the commit log,
> > > it's very hard to figure out what improved for users. A halfway widely
> > > applicable performance improvement is far more impactful than many of the
> > > feature changes we do list in the release notes.
> >
> > The practical reason this matters to users is that they want to change
> > their configuration or expectations in response to performance
> > improvements.
> >
> > And also, as Jelte mentions upthread, describing performance
> > improvements made each release in Postgres makes it clear that we are
> > consistently improving it.
> >
> > > For another, it's also very frustrating for developers that focus on
> > > performance. The reticence to note their work, while noting other, far
> > > smaller, things in the release notes, pretty much tells us that our work isn't
> > > valued.
> >
> > +many
>
> Please see the email I just posted. There are three goals we have to
> adjust for:
>
> 1. short release notes so they are readable
> 2. giving people credit for performance improvements
> 3. showing people Postgres cares about performance
I agree with all three of these goals. I would even add to 3 "show
users Postgres is addressing their performance complaints". That in
particular makes me less excited about having a "generic performance
release note item saying performance has been improved in the
following areas" (from your other email). I think that describing the
specific performance improvements is required to 1) allow users to
change expectations and configurations to take advantage of the
performance enhancements 2) ensure users know that their performance
concerns are being addressed.
> I would like to achieve 2 & 3 without harming #1. My experience is if I
> am reading a long document, and I get to a section where I start to
> wonder, "Why should I care about this?", I start to skim the rest of
> the document. I am particularly critical if I start to wonder, "Why
> does the author _think_ I should care about this?" becasue it feels like
> the author is writing for him/herself and not the audience.
I see what you are saying. We don't want to just end up with the whole
git log in the release notes. However, we know that not all users will
care about the same features. As someone said somewhere else in this
thread, presumably hackers spend time on features because some users
want them.
- Melanie
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-16 13:09 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-18 15:13 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 18:35 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
@ 2024-05-20 18:40 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-20 18:40 UTC (permalink / raw)
To: Melanie Plageman <[email protected]>; +Cc: Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Mon, May 20, 2024 at 02:35:37PM -0400, Melanie Plageman wrote:
> On Sat, May 18, 2024 at 11:13 AM Bruce Momjian <[email protected]> wrote:
> > Please see the email I just posted. There are three goals we have to
> > adjust for:
> >
> > 1. short release notes so they are readable
> > 2. giving people credit for performance improvements
> > 3. showing people Postgres cares about performance
>
> I agree with all three of these goals. I would even add to 3 "show
> users Postgres is addressing their performance complaints". That in
> particular makes me less excited about having a "generic performance
> release note item saying performance has been improved in the
> following areas" (from your other email). I think that describing the
> specific performance improvements is required to 1) allow users to
> change expectations and configurations to take advantage of the
> performance enhancements 2) ensure users know that their performance
> concerns are being addressed.
Well, as you can see, doing #2 & #3 works against accomplishing #1.
> > I would like to achieve 2 & 3 without harming #1. My experience is if I
> > am reading a long document, and I get to a section where I start to
> > wonder, "Why should I care about this?", I start to skim the rest of
> > the document. I am particularly critical if I start to wonder, "Why
> > does the author _think_ I should care about this?" becasue it feels like
> > the author is writing for him/herself and not the audience.
>
> I see what you are saying. We don't want to just end up with the whole
> git log in the release notes. However, we know that not all users will
> care about the same features. As someone said somewhere else in this
> thread, presumably hackers spend time on features because some users
> want them.
I think we need as a separate section about performance improvements
that don't affect specific workloads. Peter Eisentraut created an
Acknowledgements section at the bottom of the release notes, similar to
#2 above, so maybe someone else can add a performance internals section
too.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-16 13:09 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-18 15:13 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-21 16:40 ` Andres Freund <[email protected]>
2024-05-22 22:33 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
1 sibling, 1 reply; 194+ messages in thread
From: Andres Freund @ 2024-05-21 16:40 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Melanie Plageman <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
Hi,
On 2024-05-18 11:13:54 -0400, Bruce Momjian wrote:
> Please see the email I just posted. There are three goals we have to
> adjust for:
>
> 1. short release notes so they are readable
> 2. giving people credit for performance improvements
> 3. showing people Postgres cares about performance
>
> I would like to achieve 2 & 3 without harming #1. My experience is if I
> am reading a long document, and I get to a section where I start to
> wonder, "Why should I care about this?", I start to skim the rest of
> the document.
I agree keeping things reasonably short is important. But I don't think you're
evenly applying it as a goal.
Just skimming the notes from the end, I see
- an 8 entries long pg_stat_statements section
- multiple entries about "Create custom wait events for ..."
- three entries about adding --all to {reindexdb,vacuumdb,clusterdb}.
- an entry about adding long options to pg_archivecleanup
- two entries about grantable maintenance rights, once via pg_maintain, once
per-table
- separate entries about pg_stat_reset_slru(), pg_stat_reset_shared("slru"),
If you're concerned about brevity, we can make things shorter without skipping
over most performance imporvements.
> I am particularly critical if I start to wonder, "Why
> does the author _think_ I should care about this?" becasue it feels like
> the author is writing for him/herself and not the audience.
FWIW, I think it's a good thing for somebody other than the author to have a
hand in writing a release notes entry for a change. The primary author(s) are
often too deep into some issue to have a good view of the right level of
detail and understandability.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-16 13:09 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-18 15:13 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:40 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-22 22:33 ` Bruce Momjian <[email protected]>
2024-05-24 18:23 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-22 22:33 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Melanie Plageman <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Tue, May 21, 2024 at 09:40:28AM -0700, Andres Freund wrote:
> Hi,
>
> On 2024-05-18 11:13:54 -0400, Bruce Momjian wrote:
> > Please see the email I just posted. There are three goals we have to
> > adjust for:
> >
> > 1. short release notes so they are readable
> > 2. giving people credit for performance improvements
> > 3. showing people Postgres cares about performance
> >
> > I would like to achieve 2 & 3 without harming #1. My experience is if I
> > am reading a long document, and I get to a section where I start to
> > wonder, "Why should I care about this?", I start to skim the rest of
> > the document.
>
> I agree keeping things reasonably short is important. But I don't think you're
> evenly applying it as a goal.
>
> Just skimming the notes from the end, I see
> - an 8 entries long pg_stat_statements section
What item did you want to remove? Those are all user-visible changes.
> - multiple entries about "Create custom wait events for ..."
Well, those are all in different sections, so how can they be merged,
unless I create a "wait event section", I guess.
> - three entries about adding --all to {reindexdb,vacuumdb,clusterdb}.
The problem with merging these is that the "Specifically, --all can now
be used with" is different for all three of them.
> - an entry about adding long options to pg_archivecleanup
Well, that is a user-visible change. Should it not be listed?
> - two entries about grantable maintenance rights, once via pg_maintain, once
> per-table
Well, one is a GRANT and another is a role, so merging them seemed like
it would be too confusing.
> - separate entries about pg_stat_reset_slru(), pg_stat_reset_shared("slru"),
They are different functions with different detail text.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-16 13:09 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-18 15:13 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:40 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:33 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-24 18:23 ` Andres Freund <[email protected]>
2024-05-26 03:41 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Andres Freund @ 2024-05-24 18:23 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Melanie Plageman <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
Hi,
On 2024-05-22 18:33:03 -0400, Bruce Momjian wrote:
> On Tue, May 21, 2024 at 09:40:28AM -0700, Andres Freund wrote:
> > On 2024-05-18 11:13:54 -0400, Bruce Momjian wrote:
> > I agree keeping things reasonably short is important. But I don't think you're
> > evenly applying it as a goal.
> >
> > Just skimming the notes from the end, I see
> > - an 8 entries long pg_stat_statements section
>
> What item did you want to remove? Those are all user-visible changes.
My point here was not that we necessarily need to remove those, but that their
impact to users is smaller than many of the performance impacts you disregard.
> > - multiple entries about "Create custom wait events for ..."
>
> Well, those are all in different sections, so how can they be merged,
> unless I create a "wait event section", I guess.
They're not, all are in "Additional Modules". Instead of
- Create custom wait events for postgres_fdw (Masahiro Ikeda)
- Create custom wait events for dblink (Masahiro Ikeda)
- Allow extensions to define custom wait events (Masahiro Ikeda)
I'd make it:
- Allow extensions to define custom wait events and create custom wait events
for postgres_fdw, dblink (Masahiro Ikeda)
> > - three entries about adding --all to {reindexdb,vacuumdb,clusterdb}.
>
> The problem with merging these is that the "Specifically, --all can now
> be used with" is different for all three of them.
You said you were worried about the length of the release notes, because it
discourages users from actually reading the release notes, due to getting
bored. Having three instance of almost the same entry, with just minor changes
between them, seems to precisely endanger boring readers.
I'd probably just go for
- Add --all option to clusterdb, reindexdb, vacuumdb to process objects in all
databases matching a pattern (Nathan Bossart)
or such. The precise details of how the option works for the different
commands doesn't need to be stated in the release notes, that's more of a
reference documentation thing. But if you want to include it, we can do
something like
Specifically, --all can now be used with --table (all commands), --schema
(reindexdb, vacuumdb), and --exclude-schema (reindexdb, vacuumdb).
> > - an entry about adding long options to pg_archivecleanup
>
> Well, that is a user-visible change. Should it not be listed?
If you are concerned about the length of the release notes and as a
consequence not including more impactful performance changes, then no, it
shouldn't. It doesn't break anyones current scripts, it doesn't enable
anything new.
> > - two entries about grantable maintenance rights, once via pg_maintain, once
> > per-table
>
> Well, one is a GRANT and another is a role, so merging them seemed like
> it would be too confusing.
I don't think it has to be.
Maybe something roughly like
- Allow granting the right to perform maintenance operations (Nathan Bossart)
The permission can be granted on a per-table basis using the MAINTAIN
privilege and on a system wide basis via the pg_maintain role.
Operations that can be controlled are VACUUM, ANALYZE, REINDEX, REFRESH
MATERIALIZED VIEW, CLUSTER, and LOCK TABLE.
I'm again mostly reacting to your concern that the release notes are getting
too boring to read. Repeated content, like in the current formulation, imo
does endanger that. Current it is:
- Add per-table GRANT permission MAINTAIN to control maintenance operations (Nathan Bossart)
The operations are VACUUM, ANALYZE, REINDEX, REFRESH MATERIALIZED VIEW, CLUSTER, and LOCK TABLE.
- Add user-grantable role pg_maintain to control maintenance operations (Nathan Bossart)
The operations are VACUUM, ANALYZE, REINDEX, REFRESH MATERIALIZED VIEW, CLUSTER, and LOCK TABLE.
> > - separate entries about pg_stat_reset_slru(), pg_stat_reset_shared("slru"),
>
> They are different functions with different detail text.
So what? You can change their text. Making it three entries makes it harder
for a reader that doesn't care about resetting stats to skip over the details.
Make it something like
- Improve control over resetting statistics (Atsushi Torikoshi, Bharath
Rupireddy)
pg_stat_reset_shared() can now reset all shared statistics, by passing NULL;
pg_stat_reset_shared(NULL) also resets SLRU statistics;
pg_stat_reset_shared("slru") resets SLRU statistics, which was already
possible using pg_stat_reset_slru(NULL).
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-16 13:09 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-18 15:13 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:40 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:33 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 18:23 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-26 03:41 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-26 03:41 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Melanie Plageman <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Fri, May 24, 2024 at 11:23:29AM -0700, Andres Freund wrote:
> Hi,
>
> On 2024-05-22 18:33:03 -0400, Bruce Momjian wrote:
> > On Tue, May 21, 2024 at 09:40:28AM -0700, Andres Freund wrote:
> > > On 2024-05-18 11:13:54 -0400, Bruce Momjian wrote:
> > > I agree keeping things reasonably short is important. But I don't think you're
> > > evenly applying it as a goal.
> > >
> > > Just skimming the notes from the end, I see
> > > - an 8 entries long pg_stat_statements section
> >
> > What item did you want to remove? Those are all user-visible changes.
>
> My point here was not that we necessarily need to remove those, but that their
> impact to users is smaller than many of the performance impacts you disregard.
I liked all your detailed suggestions so applied the attached patch.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (5.4K, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 891678cc94b..1e65e99f2b2 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -628,45 +628,20 @@ Relevant columns have been removed from pg_stat_bgwriter and added to this new s
<!--
Author: Michael Paquier <[email protected]>
2023-11-12 [23c8c0c8f] Add ability to reset all shared stats types in pg_stat_r
--->
-
-<listitem>
-<para>
-Allow pg_stat_reset_shared() to reset all shared statistics (Atsushi Torikoshi)
-</para>
-
-<para>
-This is done by passing NULL.
-</para>
-</listitem>
-
-<!--
Author: Michael Paquier <[email protected]>
2023-11-16 [2e8a0edc2] Add target "slru" to pg_stat_reset_shared()
--->
-
-<listitem>
-<para>
-Allow pg_stat_reset_shared('slru') to clear SLRU statistics (Atsushi Torikoshi)
-</para>
-
-<para>
-Now pg_stat_reset_shared(NULL) also resets SLRU statistics.
-</para>
-</listitem>
-
-<!--
Author: Michael Paquier <[email protected]>
2023-11-14 [e5cca6288] Add support for pg_stat_reset_slru without argument
-->
<listitem>
<para>
-Allow pg_stat_reset_slru() to reset all SLRU statistics (Bharath Rupireddy)
+Improve control over resetting statistics (Atsushi Torikoshi, Bharath Rupireddy)
</para>
<para>
-The command pg_stat_reset_slru(NULL) already did this.
+Allow pg_stat_reset_shared() (with no arguments) and pg_stat_reset_shared(NULL) to reset all shared statistics.
+Allow pg_stat_reset_shared('slru') and pg_stat_reset_slru() (with no arguments) to reset SLRU statistics, which was already possible with pg_stat_reset_slru(NULL).
</para>
</listitem>
@@ -784,21 +759,22 @@ Add server variable trace_connection_negotiation to allow debugging of connectio
<!--
Author: Nathan Bossart <[email protected]>
2024-03-13 [ecb0fd337] Reintroduce MAINTAIN privilege and pg_maintain predefine
+Author: Nathan Bossart <[email protected]>
+2024-03-13 [ecb0fd337] Reintroduce MAINTAIN privilege and pg_maintain predefine
-->
<listitem>
<para>
-Add per-table GRANT permission MAINTAIN to control maintenance operations (Nathan Bossart)
+Allow granting the right to perform maintenance operations (Nathan Bossart)
</para>
<para>
-The operations are VACUUM, ANALYZE, REINDEX, REFRESH MATERIALIZED VIEW, CLUSTER, and LOCK TABLE.
+The permission can be granted on a per-table basis using the MAINTAIN privilege and on a per-role basis via the pg_maintain predefined role. Permitted operations are VACUUM, ANALYZE,
+REINDEX, REFRESH MATERIALIZED VIEW, CLUSTER, and LOCK TABLE.
</para>
</listitem>
<!--
-Author: Nathan Bossart <[email protected]>
-2024-03-13 [ecb0fd337] Reintroduce MAINTAIN privilege and pg_maintain predefine
-->
<listitem>
@@ -2234,45 +2210,19 @@ Allow reindexdb --index to process indexes from different tables in parallel (Ma
<!--
Author: Nathan Bossart <[email protected]>
2024-03-11 [24c928ad9] reindexdb: Allow specifying objects to process in all da
--->
-
-<listitem>
-<para>
-Allow reindexdb to process objects in all databases matching a pattern (Nathan Bossart)
-</para>
-
-<para>
-Specifically, --all can now be used with --table, --schema, --index, and --system.
-</para>
-</listitem>
-
-<!--
Author: Nathan Bossart <[email protected]>
2024-03-11 [648928c79] vacuumdb: Allow specifying objects to process in all dat
--->
-
-<listitem>
-<para>
-Allow vacuumdb to process objects in all databases matching a pattern (Nathan Bossart)
-</para>
-
-<para>
-Specifically, --all can now be used with --table, --schema, and --exclude-schema.
-</para>
-</listitem>
-
-<!--
Author: Nathan Bossart <[email protected]>
2024-03-11 [1b49d56d3] clusterdb: Allow specifying tables to process in all dat
-->
<listitem>
<para>
-Allow clusterdb to process objects in all databases matching a pattern (Nathan Bossart)
+Allow reindexdb, vacuumdb, and clusterdb to process objects in all databases matching a pattern (Nathan Bossart)
</para>
<para>
-Specifically, --all can now be used with --table.
+The new option --all controls this behavior.
</para>
</listitem>
@@ -2550,28 +2500,6 @@ This value is used by the optimizer.
</para>
</listitem>
-<!--
-Author: Michael Paquier <[email protected]>
-2023-10-05 [d61f2538a] postgres_fdw: Replace WAIT_EVENT_EXTENSION with custom w
--->
-
-<listitem>
-<para>
-Create custom wait events for postgres_fdw (Masahiro Ikeda)
-</para>
-</listitem>
-
-<!--
-Author: Michael Paquier <[email protected]>
-2023-10-05 [c789f0f6c] dblink: Replace WAIT_EVENT_EXTENSION with custom wait ev
--->
-
-<listitem>
-<para>
-Create custom wait events for dblink (Masahiro Ikeda)
-</para>
-</listitem>
-
<!--
Author: Noah Misch <[email protected]>
2024-01-08 [d3c5f37dd] Make dblink interruptible, via new libpqsrv APIs.
@@ -2696,12 +2624,20 @@ Author: Michael Paquier <[email protected]>
2023-07-31 [c9af05465] Support custom wait events for wait event type "Extensio
Author: Michael Paquier <[email protected]>
2023-10-04 [c8e318b1b] worker_spi: Rename custom wait event to "WorkerSpiMain"
+Author: Michael Paquier <[email protected]>
+2023-10-05 [d61f2538a] postgres_fdw: Replace WAIT_EVENT_EXTENSION with custom w
+Author: Michael Paquier <[email protected]>
+2023-10-05 [c789f0f6c] dblink: Replace WAIT_EVENT_EXTENSION with custom wait ev
-->
<listitem>
<para>
Allow extensions to define custom wait events (Masahiro Ikeda)
</para>
+
+<para>
+Custom wait events have been added to postgres_fdw and dblink.
+</para>
</listitem>
<!--
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-16 14:55 ` Peter Geoghegan <[email protected]>
4 siblings, 0 replies; 194+ messages in thread
From: Peter Geoghegan @ 2024-05-16 14:55 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Bruce Momjian <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Wed, May 15, 2024 at 11:48 PM Andres Freund <[email protected]> wrote:
> On 2024-05-15 10:38:20 +0200, Alvaro Herrera wrote:
> > I disagree with this. IMO the impact of the Sawada/Naylor change is
> > likely to be enormous for people with large tables and large numbers of
> > tuples to clean up (I know we've had a number of customers in this
> > situation, I can't imagine any Postgres service provider that doesn't).
> > The fact that maintenance_work_mem is no longer capped at 1GB is very
> > important and I think we should mention that explicitly in the release
> > notes, as setting it higher could make a big difference in vacuum run
> > times.
>
> +many.
TIDStore/the lifting of the maintenance_work_mem cap is likely to make
the performance of VACUUM a lot more predictable, overall. While most
VACUUM operations don't hit the limit, the limit is disproportionately
involved in cases where (for whatever reason) vacuuming becomes a long
and painful process. Even if you as a user never run into such a
problem, you still spend time worrying about it, and/or taking
measures to make sure it doesn't affect you.
The justification for not including mention of these items is that
they're not very relevant to users. I find that hard to square with
what does get included. For example, the "Source Code" section is full
of highly niche items. Items that are low impact, even for users
that'll benefit the most. Also, "Monitoring" often mentions monitoring
improvements that expose low-level implementation details (e.g. SLRU
statistics), even though there's a good chance that Bruce wouldn't
include an item for some improvement to the SLRU subsystem itself.
If somebody puts in an enormous amount of effort to get a big
performance improvement over the line, then ISTM that that effort is a
useful signal when the time comes to write the release notes (at least
up to a point). For example, Masahiko and John spent about 2 years on
the TIDStore thing, on and off. These things do not happen in a vacuum
(no pun intended). Common sense tells me that they went to those
lengths precisely because they understood that it very much was
relevant to users. That belief would have been reinforced by both
experience, and by discussion on the list during the development of
the feature.
To be fair to Bruce, it probably really is true that most individual
users won't care about (say) TIDStore. But it's probably also true
that most individual users don't care about the release notes, or at
most skim the major items.
--
Peter Geoghegan
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-18 14:59 ` Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
4 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-18 14:59 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Wed, May 15, 2024 at 08:48:02PM -0700, Andres Freund wrote:
> Hi,
>
> On 2024-05-15 10:38:20 +0200, Alvaro Herrera wrote:
> > I disagree with this. IMO the impact of the Sawada/Naylor change is
> > likely to be enormous for people with large tables and large numbers of
> > tuples to clean up (I know we've had a number of customers in this
> > situation, I can't imagine any Postgres service provider that doesn't).
> > The fact that maintenance_work_mem is no longer capped at 1GB is very
> > important and I think we should mention that explicitly in the release
> > notes, as setting it higher could make a big difference in vacuum run
> > times.
>
> +many.
>
> We're having this debate every release. I think the ongoing reticence to note
> performance improvements in the release notes is hurting Postgres.
>
> For one, performance improvements are one of the prime reason users
> upgrade. Without them being noted anywhere more dense than the commit log,
> it's very hard to figure out what improved for users. A halfway widely
> applicable performance improvement is far more impactful than many of the
> feature changes we do list in the release notes.
I agree the impact of performance improvements are often greater than
the average release note item. However, if people expect Postgres to be
faster, is it important for them to know _why_ it is faster?
If we add a new flag to a command, people will want to know about it so
they can make use of it, or if there is a performance improvement that
allows new workloads, they will want to know about that too so they can
consider those workloads.
On the flip side, a performance improvement that makes everything 10%
faster has little behavioral change for users, and in fact I think we
get ~6% faster in every major release.
> For another, it's also very frustrating for developers that focus on
> performance. The reticence to note their work, while noting other, far
> smaller, things in the release notes, pretty much tells us that our work isn't
> valued.
Yes, but are we willing to add text that every user will have to read
just for this purpose?
One think we _could_ do is to create a generic performance release note
item saying performance has been improved in the following areas, with
no more details, but we can list the authors on the item.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-21 16:27 ` Andres Freund <[email protected]>
2024-05-21 16:51 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-21 17:06 ` Re: First draft of PG 17 release notes Peter Geoghegan <[email protected]>
2024-05-21 17:50 ` Re: First draft of PG 17 release notes Robert Haas <[email protected]>
2024-05-22 22:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 4 replies; 194+ messages in thread
From: Andres Freund @ 2024-05-21 16:27 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
Hi,
On 2024-05-18 10:59:47 -0400, Bruce Momjian wrote:
> On Wed, May 15, 2024 at 08:48:02PM -0700, Andres Freund wrote:
> > +many.
> >
> > We're having this debate every release. I think the ongoing reticence to note
> > performance improvements in the release notes is hurting Postgres.
> >
> > For one, performance improvements are one of the prime reason users
> > upgrade. Without them being noted anywhere more dense than the commit log,
> > it's very hard to figure out what improved for users. A halfway widely
> > applicable performance improvement is far more impactful than many of the
> > feature changes we do list in the release notes.
>
> I agree the impact of performance improvements are often greater than
> the average release note item. However, if people expect Postgres to be
> faster, is it important for them to know _why_ it is faster?
Yes, it very often is. Performance improvements typically aren't "make
everything 3% faster", they're more "make this special thing 20%
faster". Without know what got faster, users don't know if
a) the upgrade will improve their production situation
b) they need to change something to take advantage of the improvement
> On the flip side, a performance improvement that makes everything 10%
> faster has little behavioral change for users, and in fact I think we
> get ~6% faster in every major release.
I cannot recall many "make everything faster" improvements, if any.
And even if it's "make everything faster" - that's useful for users to know,
it might solve their production problem! It's also good for PR.
Given how expensive postgres upgrades still are, we can't expect production
workloads to upgrade to every major version. The set of performance
improvements and feature additions between major versions can help users make
an informed decision.
Also, the release notes are also not just important to users. I often go back
and look in the release notes to see when some some important change was made,
because sometimes it's harder to find it in the git log, due to sheer
volume. And even just keeping up with all the changes between two releases is
hard, it's useful to be able to read the release notes and see what happened.
> > For another, it's also very frustrating for developers that focus on
> > performance. The reticence to note their work, while noting other, far
> > smaller, things in the release notes, pretty much tells us that our work isn't
> > valued.
>
> Yes, but are we willing to add text that every user will have to read
> just for this purpose?
Of course it's a tradeoff. We shouldn't verbosely note down every small
changes just because of the recognition, that'd make the release notes
unreadable. And it'd just duplicate the commit log. But that's not the same as
defaulting to not noting performance improvements, even if the performance
improvement is more impactful than many other features that are noted.
> One think we _could_ do is to create a generic performance release note
> item saying performance has been improved in the following areas, with
> no more details, but we can list the authors on the item.
To me that's the "General Performance" section. If somebody reading the
release notes doesn't care about performance, they can just skip that section
([1]). I don't see why we wouldn't want to include the same level of detail
as for other changes.
Greetings,
Andres Freund
[1] I've wondered if we should have one more level of TOC on the release note
page, so it's easier to jump to specific sections.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-21 16:51 ` Andres Freund <[email protected]>
2024-05-21 16:55 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-22 22:05 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
3 siblings, 2 replies; 194+ messages in thread
From: Andres Freund @ 2024-05-21 16:51 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
Hi,
On 2024-05-21 09:27:20 -0700, Andres Freund wrote:
> Also, the release notes are also not just important to users. I often go back
> and look in the release notes to see when some some important change was made,
> because sometimes it's harder to find it in the git log, due to sheer
> volume. And even just keeping up with all the changes between two releases is
> hard, it's useful to be able to read the release notes and see what happened.
>
> [...]
>
> [1] I've wondered if we should have one more level of TOC on the release note
> page, so it's easier to jump to specific sections.
Which reminds me: Eventually I'd like to add links to the most important
commits related to release note entries. We already do much of the work of
building that list of commits for each entry. That'd allow a reader to find
more details if interested.
Right one either has to open the sgml file (which no user will know to do), or
find the git entries manually. The latter of which is often hard, because the
git commits often will use different wording etc.
Admittedly doing so within the constraints of docbook and not wanting to
overly decrease density (both in .sgml and the resulting html) isn't a trivial
task.
That's entirely independent of my concern around noting performance
improvements in the release notes, of course.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-21 16:51 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-21 16:55 ` Alvaro Herrera <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Alvaro Herrera @ 2024-05-21 16:55 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Bruce Momjian <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On 2024-May-21, Andres Freund wrote:
> Which reminds me: Eventually I'd like to add links to the most important
> commits related to release note entries. We already do much of the work of
> building that list of commits for each entry. That'd allow a reader to find
> more details if interested.
+1. Several times I have had to open the SGML to fetch a commit ID and
build a URL to provide to someone.
--
Álvaro Herrera Breisgau, Deutschland — https://www.EnterpriseDB.com/
"Puedes vivir sólo una vez, pero si lo haces bien, una vez es suficiente"
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-21 16:51 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-22 22:05 ` Bruce Momjian <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-22 22:05 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Tue, May 21, 2024 at 09:51:09AM -0700, Andres Freund wrote:
> Hi,
>
> On 2024-05-21 09:27:20 -0700, Andres Freund wrote:
> > Also, the release notes are also not just important to users. I often go back
> > and look in the release notes to see when some some important change was made,
> > because sometimes it's harder to find it in the git log, due to sheer
> > volume. And even just keeping up with all the changes between two releases is
> > hard, it's useful to be able to read the release notes and see what happened.
> >
> > [...]
> >
> > [1] I've wondered if we should have one more level of TOC on the release note
> > page, so it's easier to jump to specific sections.
>
> Which reminds me: Eventually I'd like to add links to the most important
> commits related to release note entries. We already do much of the work of
> building that list of commits for each entry. That'd allow a reader to find
> more details if interested.
Yes, it would be cool if they could mouse-over a graphic next to each
release note item to get a popup to the commits.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-21 17:06 ` Peter Geoghegan <[email protected]>
3 siblings, 0 replies; 194+ messages in thread
From: Peter Geoghegan @ 2024-05-21 17:06 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Bruce Momjian <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Tue, May 21, 2024 at 12:27 PM Andres Freund <[email protected]> wrote:
> > I agree the impact of performance improvements are often greater than
> > the average release note item. However, if people expect Postgres to be
> > faster, is it important for them to know _why_ it is faster?
>
> Yes, it very often is.
Is it important for them to even read the release notes?
Bruce's arguments against listing performance items more often/with
greater prominence could just as easily be applied to other types of
features, in other areas. Performance is a feature (or a feature
category) -- no better or worse than any other category of feature.
> Performance improvements typically aren't "make
> everything 3% faster", they're more "make this special thing 20%
> faster". Without know what got faster, users don't know if
> a) the upgrade will improve their production situation
> b) they need to change something to take advantage of the improvement
Another important category of performance improvement is "make the
thing that was just unusable usable, for the first time ever".
Sometimes the baseline is unreasonably slow, so an improvement
effectively allows you as a user to do something that just wasn't
possible on previous versions. Other times it's addressed at something
that was very scary, like VACUUMs that need multiple rounds of index
vacuuming. Multiple rounds of index vacuuming are just woefully,
horribly inefficient, and are the single individual thing that can
make things far worse. Even if you didn't technically have that
problem before now, you did have the problem of having to worry about
it. So the work in question has sanded-down this really nasty sharp
edge. That's a substantial quality of life improvement for many users.
In short, many individual performance improvements are best thought of
as qualitative improvements, rather than quantitative improvements.
It doesn't help that there is a kind of pressure to present them as
quantitative improvements. For example, I was recently encouraged to
present my own Postgres 17 B-Tree work internally using some kind of
headline grabbing measure like "6x faster". That just seems silly to
me. I can contrive a case where it's faster by an arbitrarily large
amount. Much like how a selective index scan can be arbitrarily faster
than a sequential scan. Again, a qualitative improvement.
--
Peter Geoghegan
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-21 17:50 ` Robert Haas <[email protected]>
2024-05-21 18:26 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-22 22:13 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
3 siblings, 2 replies; 194+ messages in thread
From: Robert Haas @ 2024-05-21 17:50 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Bruce Momjian <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Tue, May 21, 2024 at 12:27 PM Andres Freund <[email protected]> wrote:
> To me that's the "General Performance" section. If somebody reading the
> release notes doesn't care about performance, they can just skip that section
> ([1]). I don't see why we wouldn't want to include the same level of detail
> as for other changes.
I'm relatively sure that we've had this argument in previous years and
essentially everyone but Bruce has agreed with the idea that
performance changes ought to be treated the same as any other kind of
improvement. The difficulty is that Bruce is the one doing the release
notes. I think it might help if someone were willing to prepare a
patch showing what they think specifically should be changed. Or maybe
Bruce would be willing to provide a list of all of the performance
improvements he doesn't think are worth release-noting or isn't sure
how to release-note, and someone else can then have a go at them.
Personally, I suspect that a part of the problem, other than the
inevitable fact that the person doing the work has a perspective on
how the work should be done with which not everyone will agree, is
that a lot of performance changes have commit messages that don't
really explain what the user impact is. For instance, consider
6dbb490261a6170a3fc3e326c6983ad63e795047 ("Combine freezing and
pruning steps in VACUUM"). It does actually say what the benefit is
("That reduces the overall amount of WAL generated") but the reader
could easily be left wondering whether that is really the selling
point. Does it also reduce CPU consumption? Is that more or less
important than the WAL reduction? Was the WAL reduction the motivation
for the work? Is the WAL reduction significant enough that this is a
feature in its own right, or is this just preparatory to some other
work? These kinds of ambiguities can exist for any commit, not just
performance commits, but I bet that on average the problem is worse
for performance-related commits.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-21 17:50 ` Re: First draft of PG 17 release notes Robert Haas <[email protected]>
@ 2024-05-21 18:26 ` Melanie Plageman <[email protected]>
2024-05-21 18:53 ` Re: First draft of PG 17 release notes Robert Haas <[email protected]>
2024-05-22 22:15 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
1 sibling, 2 replies; 194+ messages in thread
From: Melanie Plageman @ 2024-05-21 18:26 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Andres Freund <[email protected]>; Bruce Momjian <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Tue, May 21, 2024 at 1:51 PM Robert Haas <[email protected]> wrote:
>
> On Tue, May 21, 2024 at 12:27 PM Andres Freund <[email protected]> wrote:
> > To me that's the "General Performance" section. If somebody reading the
> > release notes doesn't care about performance, they can just skip that section
> > ([1]). I don't see why we wouldn't want to include the same level of detail
> > as for other changes.
>
> I'm relatively sure that we've had this argument in previous years and
> essentially everyone but Bruce has agreed with the idea that
> performance changes ought to be treated the same as any other kind of
> improvement. The difficulty is that Bruce is the one doing the release
> notes. I think it might help if someone were willing to prepare a
> patch showing what they think specifically should be changed. Or maybe
> Bruce would be willing to provide a list of all of the performance
> improvements he doesn't think are worth release-noting or isn't sure
> how to release-note, and someone else can then have a go at them.
>
> Personally, I suspect that a part of the problem, other than the
> inevitable fact that the person doing the work has a perspective on
> how the work should be done with which not everyone will agree, is
> that a lot of performance changes have commit messages that don't
> really explain what the user impact is. For instance, consider
> 6dbb490261a6170a3fc3e326c6983ad63e795047 ("Combine freezing and
> pruning steps in VACUUM"). It does actually say what the benefit is
> ("That reduces the overall amount of WAL generated") but the reader
> could easily be left wondering whether that is really the selling
> point. Does it also reduce CPU consumption? Is that more or less
> important than the WAL reduction? Was the WAL reduction the motivation
> for the work? Is the WAL reduction significant enough that this is a
> feature in its own right, or is this just preparatory to some other
> work? These kinds of ambiguities can exist for any commit, not just
> performance commits, but I bet that on average the problem is worse
> for performance-related commits.
In Postgres development, we break larger projects into smaller ones
and then those smaller projects into multiple individual commits. Each
commit needs to stand alone and each subproject needs to have a
defensible benefit. One thing that is harder with performance-related
work than non-performance feature work is that there isn't always a
final "turn it on" commit. For example, let's say you are adding a new
view that tracks new stats of some kind. You do a bunch of refactoring
and small subprojects to make it possible to add the view. Then the
final commit that actually creates the view has obvious user value to
whoever is reading the log. For performance features, it doesn't
always work like this.
For the vacuum WAL volume reduction, there were a bunch of smaller
projects throughout the last development year that I worked on that
were committed by different people and with different individual
benefits. Some changes caused vacuum to do less visibility checks (so
less CPU usage), some changed WAL format in a way that saves some
space, and some, like the commit you mention, make vacuum emit less
WAL. That commit by itself doesn't contain all of the user benefits of
the whole project. I couldn't think of a good place to list all of the
commits together that were part of the same project. Perhaps you could
argue that they were not in fact part of the same project and instead
were just small individual changes -- none of which are individually
worth including in the release notes.
- Melanie
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-21 17:50 ` Re: First draft of PG 17 release notes Robert Haas <[email protected]>
2024-05-21 18:26 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
@ 2024-05-21 18:53 ` Robert Haas <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Robert Haas @ 2024-05-21 18:53 UTC (permalink / raw)
To: Melanie Plageman <[email protected]>; +Cc: Andres Freund <[email protected]>; Bruce Momjian <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Tue, May 21, 2024 at 2:26 PM Melanie Plageman
<[email protected]> wrote:
> For the vacuum WAL volume reduction, there were a bunch of smaller
> projects throughout the last development year that I worked on that
> were committed by different people and with different individual
> benefits. Some changes caused vacuum to do less visibility checks (so
> less CPU usage), some changed WAL format in a way that saves some
> space, and some, like the commit you mention, make vacuum emit less
> WAL. That commit by itself doesn't contain all of the user benefits of
> the whole project. I couldn't think of a good place to list all of the
> commits together that were part of the same project. Perhaps you could
> argue that they were not in fact part of the same project and instead
> were just small individual changes -- none of which are individually
> worth including in the release notes.
Yeah, I think a lot of projects have this problem in one way or
another, but I think it may be worse for performance-related projects.
I wasn't intending to knock that particular commit, just to be clear,
or the commit message. I'm just saying that sometimes summarizing the
commit log may not be as easy as we'd hope.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-21 17:50 ` Re: First draft of PG 17 release notes Robert Haas <[email protected]>
2024-05-21 18:26 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
@ 2024-05-22 22:15 ` Bruce Momjian <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-22 22:15 UTC (permalink / raw)
To: Melanie Plageman <[email protected]>; +Cc: Robert Haas <[email protected]>; Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; pgsql-hackers
On Tue, May 21, 2024 at 02:26:15PM -0400, Melanie Plageman wrote:
> In Postgres development, we break larger projects into smaller ones
> and then those smaller projects into multiple individual commits. Each
> commit needs to stand alone and each subproject needs to have a
> defensible benefit. One thing that is harder with performance-related
> work than non-performance feature work is that there isn't always a
> final "turn it on" commit. For example, let's say you are adding a new
> view that tracks new stats of some kind. You do a bunch of refactoring
> and small subprojects to make it possible to add the view. Then the
> final commit that actually creates the view has obvious user value to
> whoever is reading the log. For performance features, it doesn't
> always work like this.
>
> For the vacuum WAL volume reduction, there were a bunch of smaller
> projects throughout the last development year that I worked on that
> were committed by different people and with different individual
> benefits. Some changes caused vacuum to do less visibility checks (so
> less CPU usage), some changed WAL format in a way that saves some
> space, and some, like the commit you mention, make vacuum emit less
> WAL. That commit by itself doesn't contain all of the user benefits of
> the whole project. I couldn't think of a good place to list all of the
> commits together that were part of the same project. Perhaps you could
> argue that they were not in fact part of the same project and instead
> were just small individual changes -- none of which are individually
> worth including in the release notes.
I try and group them, but I am sure imperfectly. It is very true that
infrastucture changes that enable later commits are often missed.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-21 17:50 ` Re: First draft of PG 17 release notes Robert Haas <[email protected]>
@ 2024-05-22 22:13 ` Bruce Momjian <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-22 22:13 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Tue, May 21, 2024 at 01:50:58PM -0400, Robert Haas wrote:
> On Tue, May 21, 2024 at 12:27 PM Andres Freund <[email protected]> wrote:
> > To me that's the "General Performance" section. If somebody reading the
> > release notes doesn't care about performance, they can just skip that section
> > ([1]). I don't see why we wouldn't want to include the same level of detail
> > as for other changes.
>
> I'm relatively sure that we've had this argument in previous years and
> essentially everyone but Bruce has agreed with the idea that
> performance changes ought to be treated the same as any other kind of
> improvement. The difficulty is that Bruce is the one doing the release
> notes. I think it might help if someone were willing to prepare a
> patch showing what they think specifically should be changed. Or maybe
> Bruce would be willing to provide a list of all of the performance
> improvements he doesn't think are worth release-noting or isn't sure
> how to release-note, and someone else can then have a go at them.
Well, developers do ask why their individual commits were not listed,
and I repeat the same thing, as I have done in this thread multiple
times. You can probably look over this thread to find them, or in
previous years.
To be clear, it is performance improvements that don't have user-visible
changes or enable new workloads that I skip listing. I will also note
that don't remember any user ever finding a performance boost, and then
coming to use and asking why we didn't list it --- this release note
review process seems to add all of those.
Maybe adding a section called "Internal Performance". Here is our
General Performance contents:
E.1.3.1.3. General Performance
Allow vacuum to more efficiently remove and freeze tuples (Melanie
Plageman)
WAL traffic caused by vacuum is also more compact.
Allow vacuum to more efficiently store tuple references (Masahiko
Sawada, John Naylor)
Additionally, vacuum is no longer silently limited to one gigabyte
of memory when maintenance_work_mem or autovacuum_work_mem are higher.
Optimize vacuuming of relations with no indexes (Melanie Plageman)
Increase default vacuum_buffer_usage_limit to 2MB (Thomas Munro)
Improve performance when checking roles with many memberships
(Nathan Bossart)
Improve performance of heavily-contended WAL writes (Bharath
Rupireddy)
Improve performance when transferring large blocks of data to a
client (Melih Mutlu)
Allow the grouping of file system reads with the new system variable
io_combine_limit (Thomas Munro, Andres Freund, Melanie Plageman, Nazir
Bilal Yavuz)
Do we think more user-invisible changes should be in there?
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-22 22:04 ` Bruce Momjian <[email protected]>
2024-05-23 01:34 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
3 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-22 22:04 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Tue, May 21, 2024 at 09:27:20AM -0700, Andres Freund wrote:
> On 2024-05-18 10:59:47 -0400, Bruce Momjian wrote:
> > I agree the impact of performance improvements are often greater than
> > the average release note item. However, if people expect Postgres to be
> > faster, is it important for them to know _why_ it is faster?
>
> Yes, it very often is. Performance improvements typically aren't "make
> everything 3% faster", they're more "make this special thing 20%
> faster". Without know what got faster, users don't know if
> a) the upgrade will improve their production situation
> b) they need to change something to take advantage of the improvement
You might have seen in this thread, I do record commits that speed up
workloads that are user-visible, or specifically make new workloads
possible. I assume that covers the items above, though I have to
determine this from the commit message.
> > On the flip side, a performance improvement that makes everything 10%
> > faster has little behavioral change for users, and in fact I think we
> > get ~6% faster in every major release.
>
> I cannot recall many "make everything faster" improvements, if any.
>
> And even if it's "make everything faster" - that's useful for users to know,
> it might solve their production problem! It's also good for PR.
Again, it is down to having three goals for the release notes, and #1 is
having it readable/short, and 2 & 3 are for PR and acknowledging authors.
> Also, the release notes are also not just important to users. I often go back
> and look in the release notes to see when some some important change was made,
> because sometimes it's harder to find it in the git log, due to sheer
> volume. And even just keeping up with all the changes between two releases is
> hard, it's useful to be able to read the release notes and see what happened.
Well, I would say we need some _other_ way to record and perhaps
advertise such changes.
> > > For another, it's also very frustrating for developers that focus on
> > > performance. The reticence to note their work, while noting other, far
> > > smaller, things in the release notes, pretty much tells us that our work isn't
> > > valued.
> >
> > Yes, but are we willing to add text that every user will have to read
> > just for this purpose?
>
> Of course it's a tradeoff. We shouldn't verbosely note down every small
> changes just because of the recognition, that'd make the release notes
> unreadable. And it'd just duplicate the commit log. But that's not the same as
> defaulting to not noting performance improvements, even if the performance
> improvement is more impactful than many other features that are noted.
Again, see above, I do mention performance improvements, but they have
to be user-visible or enable new workloads.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-23 01:34 ` David Rowley <[email protected]>
2024-05-23 02:01 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: David Rowley @ 2024-05-23 01:34 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Thu, 23 May 2024 at 10:04, Bruce Momjian <[email protected]> wrote:
> You might have seen in this thread, I do record commits that speed up
> workloads that are user-visible, or specifically make new workloads
> possible. I assume that covers the items above, though I have to
> determine this from the commit message.
It sometimes is hard to write something specific in the commit message
about the actual performance increase.
For example, if a commit removes an O(N log2 N) algorithm and replaces
it with an O(1), you can't say there's an X% increase in performance
as the performance increase depends on the value of N.
Jelte did call me out for not mentioning enough detail about the
performance in c4ab7da60, but if I claimed any % of an increase, it
would have been specific to some workload.
What is the best way to communicate this stuff so it's easily
identifiable when you parse the commit messages?
David
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 01:34 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
@ 2024-05-23 02:01 ` Bruce Momjian <[email protected]>
2024-05-23 02:27 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-23 02:01 UTC (permalink / raw)
To: David Rowley <[email protected]>; +Cc: Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Thu, May 23, 2024 at 01:34:10PM +1200, David Rowley wrote:
> On Thu, 23 May 2024 at 10:04, Bruce Momjian <[email protected]> wrote:
> > You might have seen in this thread, I do record commits that speed up
> > workloads that are user-visible, or specifically make new workloads
> > possible. I assume that covers the items above, though I have to
> > determine this from the commit message.
>
> It sometimes is hard to write something specific in the commit message
> about the actual performance increase.
>
> For example, if a commit removes an O(N log2 N) algorithm and replaces
> it with an O(1), you can't say there's an X% increase in performance
> as the performance increase depends on the value of N.
>
> Jelte did call me out for not mentioning enough detail about the
> performance in c4ab7da60, but if I claimed any % of an increase, it
> would have been specific to some workload.
>
> What is the best way to communicate this stuff so it's easily
> identifiable when you parse the commit messages?
This is why I think we need an "Internal Performance" section where we
can be clear about simple scaling improvements that might have no
user-visible explanation. I would suggest putting it after our "Source
code" section.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 01:34 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-23 02:01 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-23 02:27 ` David Rowley <[email protected]>
2024-05-24 03:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: David Rowley @ 2024-05-23 02:27 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Thu, 23 May 2024 at 14:01, Bruce Momjian <[email protected]> wrote:
>
> On Thu, May 23, 2024 at 01:34:10PM +1200, David Rowley wrote:
> > What is the best way to communicate this stuff so it's easily
> > identifiable when you parse the commit messages?
>
> This is why I think we need an "Internal Performance" section where we
> can be clear about simple scaling improvements that might have no
> user-visible explanation. I would suggest putting it after our "Source
> code" section.
hmm, but that does not really answer my question. There's still a
communication barrier if you're parsing the commit messages are
committers don't clearly state some performance improvement numbers
for the reason I stated.
I also don't agree these should be left to "Source code" section. I
feel that section is best suited for extension authors who might care
about some internal API change. I'm talking of stuff that makes some
user queries possibly N times (!) faster. Surely "Source Code" isn't
the place to talk about that?
David
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 01:34 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-23 02:01 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 02:27 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
@ 2024-05-24 03:04 ` Bruce Momjian <[email protected]>
2024-05-24 03:11 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
2024-05-24 13:54 ` Re: First draft of PG 17 release notes Robert Haas <[email protected]>
0 siblings, 2 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-24 03:04 UTC (permalink / raw)
To: David Rowley <[email protected]>; +Cc: Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Thu, May 23, 2024 at 02:27:07PM +1200, David Rowley wrote:
> On Thu, 23 May 2024 at 14:01, Bruce Momjian <[email protected]> wrote:
> >
> > On Thu, May 23, 2024 at 01:34:10PM +1200, David Rowley wrote:
> > > What is the best way to communicate this stuff so it's easily
> > > identifiable when you parse the commit messages?
> >
> > This is why I think we need an "Internal Performance" section where we
> > can be clear about simple scaling improvements that might have no
> > user-visible explanation. I would suggest putting it after our "Source
> > code" section.
>
> hmm, but that does not really answer my question. There's still a
> communication barrier if you're parsing the commit messages are
> committers don't clearly state some performance improvement numbers
> for the reason I stated.
For a case where O(N^2) become O(N), we might not even know the
performance change since it is a micro-optimization. That is why I
suggested we call it "Internal Performance".
> I also don't agree these should be left to "Source code" section. I
> feel that section is best suited for extension authors who might care
> about some internal API change. I'm talking of stuff that makes some
> user queries possibly N times (!) faster. Surely "Source Code" isn't
> the place to talk about that?
I said a new section "after our 'Source code' section," not in the
source code section.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 01:34 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-23 02:01 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 02:27 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-24 03:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-24 03:11 ` Tom Lane <[email protected]>
2024-05-24 03:27 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
1 sibling, 1 reply; 194+ messages in thread
From: Tom Lane @ 2024-05-24 03:11 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: David Rowley <[email protected]>; Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
Bruce Momjian <[email protected]> writes:
> On Thu, May 23, 2024 at 02:27:07PM +1200, David Rowley wrote:
>> I also don't agree these should be left to "Source code" section. I
>> feel that section is best suited for extension authors who might care
>> about some internal API change. I'm talking of stuff that makes some
>> user queries possibly N times (!) faster. Surely "Source Code" isn't
>> the place to talk about that?
> I said a new section "after our 'Source code' section," not in the
> source code section.
Surely, if we make this a separate section, it would come before
'Source code'?
I am not sure Bruce that you realize that your disregard for
performance improvements is shared by nobody. Arguably,
performance is 90% of what we do these days, and it's also
90% of what users care about.
regards, tom lane
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 01:34 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-23 02:01 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 02:27 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-24 03:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 03:11 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
@ 2024-05-24 03:27 ` Bruce Momjian <[email protected]>
2024-05-24 17:50 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-24 03:27 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: David Rowley <[email protected]>; Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Thu, May 23, 2024 at 11:11:10PM -0400, Tom Lane wrote:
> Bruce Momjian <[email protected]> writes:
> > On Thu, May 23, 2024 at 02:27:07PM +1200, David Rowley wrote:
> >> I also don't agree these should be left to "Source code" section. I
> >> feel that section is best suited for extension authors who might care
> >> about some internal API change. I'm talking of stuff that makes some
> >> user queries possibly N times (!) faster. Surely "Source Code" isn't
> >> the place to talk about that?
>
> > I said a new section "after our 'Source code' section," not in the
> > source code section.
>
> Surely, if we make this a separate section, it would come before
> 'Source code'?
>
> I am not sure Bruce that you realize that your disregard for
> performance improvements is shared by nobody. Arguably,
> performance is 90% of what we do these days, and it's also
> 90% of what users care about.
Please stop saying I don't document performance. I have already
explained enough which performance items I choose. Please address my
criteria or suggest new criteria.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 01:34 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-23 02:01 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 02:27 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-24 03:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 03:11 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
2024-05-24 03:27 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-24 17:50 ` Andres Freund <[email protected]>
2024-05-24 18:26 ` Re: First draft of PG 17 release notes Peter Geoghegan <[email protected]>
2024-05-26 02:10 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 2 replies; 194+ messages in thread
From: Andres Freund @ 2024-05-24 17:50 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Tom Lane <[email protected]>; David Rowley <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
Hi,
On 2024-05-23 23:27:04 -0400, Bruce Momjian wrote:
> On Thu, May 23, 2024 at 11:11:10PM -0400, Tom Lane wrote:
> > Bruce Momjian <[email protected]> writes:
> > I am not sure Bruce that you realize that your disregard for
> > performance improvements is shared by nobody. Arguably,
> > performance is 90% of what we do these days, and it's also
> > 90% of what users care about.
>
> Please stop saying I don't document performance. I have already
> explained enough which performance items I choose. Please address my
> criteria or suggest new criteria.
Bruce, just about everyone seems to disagree with your current approach. And
not just this year, this has been a discussion in most if not all release note
threads of the last few years.
People, including me, *have* addressed your criteria, but you just waved those
concerns away. It's hard to continue discussing criteria when it doesn't at
all feel like a conversation.
In the end, these are patches to the source code, I don't think you can just
wave away widespread disagreement with your changes. That's not how we do
postgres development.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 01:34 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-23 02:01 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 02:27 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-24 03:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 03:11 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
2024-05-24 03:27 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 17:50 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-24 18:26 ` Peter Geoghegan <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Peter Geoghegan @ 2024-05-24 18:26 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; David Rowley <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Fri, May 24, 2024 at 1:50 PM Andres Freund <[email protected]> wrote:
> Bruce, just about everyone seems to disagree with your current approach. And
> not just this year, this has been a discussion in most if not all release note
> threads of the last few years.
+1.
> People, including me, *have* addressed your criteria, but you just waved those
> concerns away. It's hard to continue discussing criteria when it doesn't at
> all feel like a conversation.
At one point on this thread, Bruce said "I am particularly critical if
I start to wonder, "Why does the author _think_ I should care about
this?" because it feels like the author is writing for him/herself and
not the audience."
Whenever this sort of thing has come up in the past, and I pushed
back, Bruce seemed to respond along these lines: he seemed to suggest
that there was some kind of conflict of interests involved. This isn't
completely unreasonable, of course -- my motivations aren't wholly
irrelevant. But for the most part they're *not* very relevant, and
wouldn't be even if Bruce's worst suspicions were actually true. In
principle it shouldn't matter that I'm biased, if I happen to be
correct in some relevant sense.
Everybody has some kind of bias. Even if my bias in these matters was
a significant factor (which I tend to doubt), I don't think that it's
fair to suggest that it's self-serving or careerist. My bias was
probably present before I even began work on the feature in question.
I tend to work on things because I believe that they're important --
it's not the other way around (at least not to a significant degree).
> In the end, these are patches to the source code, I don't think you can just
> wave away widespread disagreement with your changes. That's not how we do
> postgres development.
In lots of cases (a large minority of cases) the problem isn't even
really with the emphasis of one type of item over another/the
inclusion or non-inclusion of some individual item. It's actually a
problem with the information being presented in the most useful way.
Often I've suggested what I believe to be a better wording on the
merits (usually less obscure and more accessible language), only to be
met with the same sort of resistance from Bruce. If I've put a huge
amount of work into the item (as is usually the case), then I think
that I am at least entitled to a fair hearing.
I don't expect Bruce to meet me halfway, or even for him to meet me a
quarter of the way -- somebody has to be empowered to say no here
(even to very senior community members). I just don't think that he
has seriously considered my feedback in this area over the years. Not
always, not consistently, but often enough for it to seem like a real
problem.
--
Peter Geoghegan
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 01:34 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-23 02:01 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 02:27 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-24 03:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 03:11 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
2024-05-24 03:27 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 17:50 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
@ 2024-05-26 02:10 ` Bruce Momjian <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-26 02:10 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; David Rowley <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Fri, May 24, 2024 at 10:50:28AM -0700, Andres Freund wrote:
> Hi,
>
> On 2024-05-23 23:27:04 -0400, Bruce Momjian wrote:
> > On Thu, May 23, 2024 at 11:11:10PM -0400, Tom Lane wrote:
> > > Bruce Momjian <[email protected]> writes:
> > > I am not sure Bruce that you realize that your disregard for
> > > performance improvements is shared by nobody. Arguably,
> > > performance is 90% of what we do these days, and it's also
> > > 90% of what users care about.
> >
> > Please stop saying I don't document performance. I have already
> > explained enough which performance items I choose. Please address my
> > criteria or suggest new criteria.
>
> Bruce, just about everyone seems to disagree with your current approach. And
> not just this year, this has been a discussion in most if not all release note
> threads of the last few years.
>
> People, including me, *have* addressed your criteria, but you just waved those
> concerns away. It's hard to continue discussing criteria when it doesn't at
> all feel like a conversation.
>
> In the end, these are patches to the source code, I don't think you can just
> wave away widespread disagreement with your changes. That's not how we do
> postgres development.
Well, let's start with a new section for PG 17 that lists these. Is it
20 items, 50, or 150? I have no idea, but without the user-visible
filter, I am unable to determine what not-included performance features
are worthy of the release notes.
Can someone do that? There is no reason other committers can't change
the release notes. Yes, I realize we are looking for a consistent
voice, but the new section can probably have its own style, and I can
make adjustments if desired.
Also, I think this has gone unaddressed so long because if we skip a
user-visible change, users complain, but I don't remember anyone
complaining about skipped performance changes.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 01:34 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-23 02:01 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 02:27 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-24 03:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-24 13:54 ` Robert Haas <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Robert Haas @ 2024-05-24 13:54 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: David Rowley <[email protected]>; Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; Melanie Plageman <[email protected]>; pgsql-hackers
On Thu, May 23, 2024 at 11:04 PM Bruce Momjian <[email protected]> wrote:
> For a case where O(N^2) become O(N), we might not even know the
> performance change since it is a micro-optimization. That is why I
> suggested we call it "Internal Performance".
I don't get this at all. If we can't tell the difference between
O(N^2) and O(N), then N was small enough that there wasn't any real
need to optimize in the first place. I think we should be assuming
that if somebody took the trouble to write a patch, the difference did
matter. Hence the change would be user-visible, and should be
documented.
"Internal Performance" doesn't make a lot of sense to me as a section
heading. What does "Internal" mean here as opposed to "General"? I
suspect you mean to imply that the user won't be able to tell the
difference, but I doubt that very much. We make performance
improvements because they are user-visible. If a performance
improvement is so miniscule that nobody would ever notice the
difference, well then I don't think it needs to be release-noted at
all, and we might have a few changes like that where people were
mostly aiming for code cleanliness. But in general, what people do is
look for something that's slow (for them) and try to make it faster.
So the presumption should be that a performance feature has a
meaningful impact on users, and then in rare cases we may decide
otherwise.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-16 02:39 ` jian he <[email protected]>
2024-05-16 02:53 ` Re: First draft of PG 17 release notes David G. Johnston <[email protected]>
2024-05-16 02:55 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 2 replies; 194+ messages in thread
From: jian he @ 2024-05-16 02:39 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
>
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
>> Add local I/O block read/write timing statistics columns of pg_stat_statement (Nazir Bilal Yavuz)
>> The new columns are "local_blk_read_time" and "local_blk_write_time".
here, "pg_stat_statement" should be "pg_stat_statements"?
>> Add optional fourth parameter to pg_stat_statements_reset() to allow for the resetting of only min/max statistics (Andrei Zubkov)
>> This parameter defaults to "false".
here, "parameter", should be "argument"?
maybe
>> Add optional fourth boolean argument (minmax_only) to pg_stat_statements_reset() to allow for the resetting of only min/max statistics (Andrei Zubkov)
>> This argument defaults to "false".
----------------------------------------------------------------
in section: E.1.2. Migration to Version 17
>> Rename I/O block read/write timing statistics columns of pg_stat_statement (Nazir Bilal Yavuz)
>> This renames "blk_read_time" to "shared_blk_read_time", and "blk_write_time" to "shared_blk_write_time".
"pg_stat_statement" should be "pg_stat_statements"?
also, we only mentioned, pg_stat_statements some columns name changed
in "E.1.2. Migration to Version 17"
but if you look at the release note pg_stat_statements section,
we added a bunch of columns, which are far more incompatibile than
just colunm name changes.
not sure we need add these in section "E.1.2. Migration to Version 17"
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 02:39 ` Re: First draft of PG 17 release notes jian he <[email protected]>
@ 2024-05-16 02:53 ` David G. Johnston <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: David G. Johnston @ 2024-05-16 02:53 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Bruce Momjian <[email protected]>; pgsql-hackers
On Wednesday, May 15, 2024, jian he <[email protected]> wrote:
> On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> >
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
> >
>
> in section: E.1.2. Migration to Version 17
>
> >> Rename I/O block read/write timing statistics columns of
> pg_stat_statement (Nazir Bilal Yavuz)
> >> This renames "blk_read_time" to "shared_blk_read_time", and
> "blk_write_time" to "shared_blk_write_time".
>
> we only mentioned, pg_stat_statements some columns name changed
> in "E.1.2. Migration to Version 17"
> but if you look at the release note pg_stat_statements section,
> we added a bunch of columns, which are far more incompatibile than
> just colunm name changes.
>
> not sure we need add these in section "E.1.2. Migration to Version 17"
>
>
New columns are not a migration issue since nothing being migrated forward
ever referenced them. Its the ones that existing code knows about that
we’ve removed (including renames) that matter from a migration perspective.
David J.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 02:39 ` Re: First draft of PG 17 release notes jian he <[email protected]>
@ 2024-05-16 02:55 ` Bruce Momjian <[email protected]>
1 sibling, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-16 02:55 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: pgsql-hackers
On Thu, May 16, 2024 at 10:39:18AM +0800, jian he wrote:
> On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> >
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
> >
>
> >> Add local I/O block read/write timing statistics columns of pg_stat_statement (Nazir Bilal Yavuz)
> >> The new columns are "local_blk_read_time" and "local_blk_write_time".
> here, "pg_stat_statement" should be "pg_stat_statements"?
Agreed.
> >> Add optional fourth parameter to pg_stat_statements_reset() to allow for the resetting of only min/max statistics (Andrei Zubkov)
> >> This parameter defaults to "false".
> here, "parameter", should be "argument"?
>
> maybe
> >> Add optional fourth boolean argument (minmax_only) to pg_stat_statements_reset() to allow for the resetting of only min/max statistics (Andrei Zubkov)
> >> This argument defaults to "false".
Sure.
> ----------------------------------------------------------------
> in section: E.1.2. Migration to Version 17
>
> >> Rename I/O block read/write timing statistics columns of pg_stat_statement (Nazir Bilal Yavuz)
> >> This renames "blk_read_time" to "shared_blk_read_time", and "blk_write_time" to "shared_blk_write_time".
>
> "pg_stat_statement" should be "pg_stat_statements"?
Yes, fixed.
> also, we only mentioned, pg_stat_statements some columns name changed
> in "E.1.2. Migration to Version 17"
> but if you look at the release note pg_stat_statements section,
> we added a bunch of columns, which are far more incompatibile than
> just colunm name changes.
>
> not sure we need add these in section "E.1.2. Migration to Version 17"
Well, new columns don't cause breakage like renamed columns, which is
why I only put renames/removed columns in the migration section.
Also, thanks everyone for the release notes feedback. In some cases I
made a mistake, and in some cases I misjudged the item.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-16 08:29 ` jian he <[email protected]>
2024-05-18 16:11 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: jian he @ 2024-05-16 08:29 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
>
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
>> Add jsonpath methods to convert JSON values to different data types (Jeevan Chalke)
>> The jsonpath methods are .bigint(), .boolean(), .date(), .decimal([precision [, scale]]), .integer(), .number(), .string(), .time(), .time_tz(), .timestamp(), and .timestamp_tz().
I think it's slightly incorrect.
for example:
select jsonb_path_query('"2023-08-15"', '$.date()');
I think it does is trying to validate json value "2023-08-15" can be a
date value, if so, print json string out, else error out.
"convert JSON values to different data types"
meaning that we are converting json values to another data type, date?
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 08:29 ` Re: First draft of PG 17 release notes jian he <[email protected]>
@ 2024-05-18 16:11 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-18 16:11 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: pgsql-hackers
On Thu, May 16, 2024 at 04:29:38PM +0800, jian he wrote:
> On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> >
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
> >
>
> >> Add jsonpath methods to convert JSON values to different data types (Jeevan Chalke)
> >> The jsonpath methods are .bigint(), .boolean(), .date(), .decimal([precision [, scale]]), .integer(), .number(), .string(), .time(), .time_tz(), .timestamp(), and .timestamp_tz().
>
> I think it's slightly incorrect.
>
> for example:
> select jsonb_path_query('"2023-08-15"', '$.date()');
> I think it does is trying to validate json value "2023-08-15" can be a
> date value, if so, print json string out, else error out.
>
>
> "convert JSON values to different data types"
> meaning that we are converting json values to another data type, date?
I see your point. I have reworded it to be:
Add jsonpath methods to convert JSON values to other JSON data
types (Jeevan Chalke)
Does that help? I think your example is causing confusion because once
JSON values enter the SQL data type space, they are strings.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-17 13:22 ` jian he <[email protected]>
2024-05-18 21:37 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: jian he @ 2024-05-17 13:22 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
>
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
> It will be improved until the final release. The item count is 188,
> which is similar to recent releases:
>
This thread mentioned performance.
actually this[1] refactored some interval aggregation related functions,
which will make these two aggregate function: avg(interval), sum(interval)
run faster, especially avg(interval).
see [2].
well, I guess, this is a kind of niche performance improvement to be
mentioned separately.
these 3 items need to be removed, because of
https://git.postgresql.org/cgit/postgresql.git/commit/?id=8aee330af55d8a759b2b73f5a771d9d34a7b887f
>> Add stratnum GiST support function (Paul A. Jungwirth)
>> Allow foreign keys to reference WITHOUT OVERLAPS primary keys (Paul A. Jungwirth)
>> The keyword PERIOD is used for this purpose.
>> Allow PRIMARY KEY and UNIQUE constraints to use WITHOUT OVERLAPS for non-overlapping exclusion constraints (Paul A. Jungwirth)
[1] https://git.postgresql.org/cgit/postgresql.git/commit/?id=519fc1bd9e9d7b408903e44f55f83f6db30742b7
[2] https://www.postgresql.org/message-id/CAEZATCUJ0xjyQUL7SHKxJ5a%2BDm5pjoq-WO3NtkDLi6c76rh58w%40mail.g...
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-17 13:22 ` Re: First draft of PG 17 release notes jian he <[email protected]>
@ 2024-05-18 21:37 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-18 21:37 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: pgsql-hackers
On Fri, May 17, 2024 at 09:22:59PM +0800, jian he wrote:
> On Thu, May 9, 2024 at 12:04 PM Bruce Momjian <[email protected]> wrote:
> >
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
> >
> > It will be improved until the final release. The item count is 188,
> > which is similar to recent releases:
> >
>
> This thread mentioned performance.
> actually this[1] refactored some interval aggregation related functions,
> which will make these two aggregate function: avg(interval), sum(interval)
> run faster, especially avg(interval).
> see [2].
> well, I guess, this is a kind of niche performance improvement to be
> mentioned separately.
>
>
> these 3 items need to be removed, because of
> https://git.postgresql.org/cgit/postgresql.git/commit/?id=8aee330af55d8a759b2b73f5a771d9d34a7b887f
>
> >> Add stratnum GiST support function (Paul A. Jungwirth)
>
> >> Allow foreign keys to reference WITHOUT OVERLAPS primary keys (Paul A. Jungwirth)
> >> The keyword PERIOD is used for this purpose.
>
> >> Allow PRIMARY KEY and UNIQUE constraints to use WITHOUT OVERLAPS for non-overlapping exclusion constraints (Paul A. Jungwirth)
>
>
> [1] https://git.postgresql.org/cgit/postgresql.git/commit/?id=519fc1bd9e9d7b408903e44f55f83f6db30742b7
> [2] https://www.postgresql.org/message-id/CAEZATCUJ0xjyQUL7SHKxJ5a%2BDm5pjoq-WO3NtkDLi6c76rh58w%40mail.g...
Agreed, I have applied the attached patch to make the release notes
current.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (2.2K, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 9c511848943..0bc1c9a14ad 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -6,7 +6,7 @@
<formalpara>
<title>Release date:</title>
- <para>2024-??-??, AS OF 2024-05-14</para>
+ <para>2024-??-??, AS OF 2024-05-18</para>
</formalpara>
<sect2 id="release-17-highlights">
@@ -480,17 +480,6 @@ Author: Tomas Vondra <[email protected]>
<para>
Allow BRIN indexes to be created using parallel workers (Tomas Vondra, Matthias van de Meent)
</para>
-</listitem>
-
-<!--
-Author: Peter Eisentraut <[email protected]>
-2024-01-19 [6db4598fc] Add stratnum GiST support function
--->
-
-<listitem>
-<para>
-Add stratnum GiST support function (Paul A. Jungwirth)
-</para>
</listitem>
</itemizedlist>
@@ -1467,34 +1456,6 @@ Add DEFAULT setting for ALTER TABLE .. SET ACCESS METHOD (Michael Paquier)
</para>
</listitem>
-<!--
-Author: Peter Eisentraut <[email protected]>
-2024-03-24 [34768ee36] Add temporal FOREIGN KEY contraints
--->
-
-<listitem>
-<para>
-Allow foreign keys to reference WITHOUT OVERLAPS primary keys (Paul A. Jungwirth)
-</para>
-
-<para>
-The keyword PERIOD is used for this purpose.
-</para>
-</listitem>
-
-<!--
-Author: Peter Eisentraut <[email protected]>
-2024-01-24 [46a0cd4ce] Add temporal PRIMARY KEY and UNIQUE constraints
-Author: Peter Eisentraut <[email protected]>
-2024-03-05 [030e10ff1] Rename pg_constraint.conwithoutoverlaps to conperiod
--->
-
-<listitem>
-<para>
-Allow PRIMARY KEY and UNIQUE constraints to use WITHOUT OVERLAPS for non-overlapping exclusion constraints (Paul A. Jungwirth)
-</para>
-</listitem>
-
<!--
Author: Alexander Korotkov <[email protected]>
2023-10-16 [e83d1b0c4] Add support event triggers on authenticated login
@@ -1957,6 +1918,8 @@ Author: Heikki Linnakangas <[email protected]>
2024-04-29 [17a834a04] Reject SSL connection if ALPN is used but there's no com
Author: Heikki Linnakangas <[email protected]>
2024-05-11 [407e0b023] Change ALPN protocol ID to IANA-approved "postgresql"
+Author: Heikki Linnakangas <[email protected]>
+2024-05-16 [fb5718f35] Remove option to fall back from direct to postgres SSL n
-->
<listitem>
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-17 13:42 ` Daniel Verite <[email protected]>
27 siblings, 0 replies; 194+ messages in thread
From: Daniel Verite @ 2024-05-17 13:42 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
Bruce Momjian wrote:
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
About the changes in collations:
<quote>
Create a "builtin" collation provider similar to libc's C locale
(Jeff Davis)
It uses a "C" locale which is identical but independent of
libc, but it allows the use of non-"C" collations like "en_US"
and "C.UTF-8" with the "C" locale, which libc does not. MORE?
</quote>
The new builtin provider has two collations:
* ucs_basic which is 100% identical to "C". It was introduced
several versions ago and the v17 novelty is simply to change
its pg_collation.collprovider from 'c' to 'b'.
* pg_c_utf8 which sorts like "C" but is Unicode-aware for
the rest, which makes it quite different from "C".
It's also different from the other UTF-8 collations that could
be used up to v17 in that it does not depend on an external
library, making it free from the collation OS-upgrade risks.
The part that is concretely of interest to users is the introduction
of pg_c_utf8. As described in [1]:
<quote>
pg_c_utf8
This collation sorts by Unicode code point values rather than
natural language order. For the functions lower, initcap, and
upper, it uses Unicode simple case mapping. For pattern
matching (including regular expressions), it uses the POSIX
Compatible variant of Unicode Compatibility Properties. Behavior
is efficient and stable within a Postgres major version. This
collation is only available for encoding UTF8.
</quote>
I'd suggest that the relnote entry should be more like a condensed
version of that description, without mentioning en_US or C.UTF-8,
whose existence and semantics are OS-dependent, contrary to pg_c_utf8.
[1] https://www.postgresql.org/docs/devel/collation.html
Best regards,
--
Daniel Vérité
https://postgresql.verite.pro/
Twitter: @DanielVerite
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-17 20:30 ` Jeff Davis <[email protected]>
2024-05-18 21:51 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Jeff Davis @ 2024-05-17 20:30 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; pgsql-hackers
On Thu, 2024-05-09 at 00:03 -0400, Bruce Momjian wrote:
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
For this item:
Create a "builtin" collation provider similar to libc's C
locale (Jeff Davis)
It uses a "C" locale which is identical but independent of
libc, but it allows the use of non-"C" collations like "en_US"
and "C.UTF-8" with the "C" locale, which libc does not. MORE?
I suggest something more like:
New, platform-independent "builtin" collation
provider. (Jeff Davis)
Currently, it offers the "C" and "C.UTF-8" locales. The
"C.UTF-8" locale combines stable and fast code point order
collation with Unicode character semantics.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-17 20:30 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
@ 2024-05-18 21:51 ` Bruce Momjian <[email protected]>
2024-05-20 18:48 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-18 21:51 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: pgsql-hackers
On Fri, May 17, 2024 at 01:30:03PM -0700, Jeff Davis wrote:
> On Thu, 2024-05-09 at 00:03 -0400, Bruce Momjian wrote:
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
>
> For this item:
>
> Create a "builtin" collation provider similar to libc's C
> locale (Jeff Davis)
>
> It uses a "C" locale which is identical but independent of
> libc, but it allows the use of non-"C" collations like "en_US"
> and "C.UTF-8" with the "C" locale, which libc does not. MORE?
>
> I suggest something more like:
>
> New, platform-independent "builtin" collation
> provider. (Jeff Davis)
>
> Currently, it offers the "C" and "C.UTF-8" locales. The
> "C.UTF-8" locale combines stable and fast code point order
> collation with Unicode character semantics.
Okay, I went with the attached applied patch. Adjustments?
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (793B, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 0bc1c9a14ad..5d2fcd7d7de 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -886,11 +886,11 @@ Author: Jeff Davis <[email protected]>
<listitem>
<para>
-Create a "builtin" collation provider similar to libc's C locale (Jeff Davis)
+Create "builtin" collation provider similar to libc's C locale (Jeff Davis)
</para>
<para>
-It uses a "C" locale which is identical but independent of libc, but it allows the use of non-"C" collations like "en_US" and "C.UTF-8" with the "C" locale, which libc does not. MORE?
+While its C locale is similar but independent of libc, its C.UTF-8 locale sorts by Unicode code points and has Unicode-based case folding.
</para>
</listitem>
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-17 20:30 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-18 21:51 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-20 18:48 ` Jeff Davis <[email protected]>
2024-05-22 22:39 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Jeff Davis @ 2024-05-20 18:48 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Sat, 2024-05-18 at 17:51 -0400, Bruce Momjian wrote:
> Okay, I went with the attached applied patch. Adjustments?
I think it should have more emphasis on the actual new feature: a
platform-independent builtin collation provider and the C.UTF-8 locale.
The user can look at the documentation for comparison with libc.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-17 20:30 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-18 21:51 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 18:48 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
@ 2024-05-22 22:39 ` Bruce Momjian <[email protected]>
2024-05-23 15:30 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-22 22:39 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: pgsql-hackers
On Mon, May 20, 2024 at 11:48:09AM -0700, Jeff Davis wrote:
> On Sat, 2024-05-18 at 17:51 -0400, Bruce Momjian wrote:
> > Okay, I went with the attached applied patch. Adjustments?
>
> I think it should have more emphasis on the actual new feature: a
> platform-independent builtin collation provider and the C.UTF-8 locale.
>
> The user can look at the documentation for comparison with libc.
Okay, changed with the attached applied patch.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (641B, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 3025f31f8a0..e381dbe5d96 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -885,11 +885,11 @@ Author: Jeff Davis <[email protected]>
<listitem>
<para>
-Create "builtin" collation provider similar to libc's C locale (Jeff Davis)
+Add a builtin platform-independent collation provider (Jeff Davis)
</para>
<para>
-While its C locale is similar but independent of libc, its C.UTF-8 locale sorts by Unicode code points and has Unicode-based case folding.
+This supports "C" and "C.UTF-8" collations.
</para>
</listitem>
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-17 20:30 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-18 21:51 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 18:48 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-22 22:39 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-23 15:30 ` Jeff Davis <[email protected]>
2024-05-23 19:54 ` Re: First draft of PG 17 release notes Marcos Pegoraro <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Jeff Davis @ 2024-05-23 15:30 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Wed, 2024-05-22 at 18:39 -0400, Bruce Momjian wrote:
> On Mon, May 20, 2024 at 11:48:09AM -0700, Jeff Davis wrote:
> > On Sat, 2024-05-18 at 17:51 -0400, Bruce Momjian wrote:
> > > Okay, I went with the attached applied patch. Adjustments?
> >
> > I think it should have more emphasis on the actual new feature: a
> > platform-independent builtin collation provider and the C.UTF-8
> > locale.
> >
> > The user can look at the documentation for comparison with libc.
>
> Okay, changed with the attached applied patch.
Thank you, looks good to me.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-17 20:30 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-18 21:51 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 18:48 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-22 22:39 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 15:30 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
@ 2024-05-23 19:54 ` Marcos Pegoraro <[email protected]>
2024-05-26 03:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Marcos Pegoraro @ 2024-05-23 19:54 UTC (permalink / raw)
To: ; +Cc: Bruce Momjian <[email protected]>; pgsql-hackers
-
Rename SLRU columns in system view pg_stat_slru (Alvaro Herrera)
The column names accepted by pg_stat_slru_rest() are also changed.
Is pg_stat_slru_rest() correct ?
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-17 20:30 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-18 21:51 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 18:48 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-22 22:39 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 15:30 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-23 19:54 ` Re: First draft of PG 17 release notes Marcos Pegoraro <[email protected]>
@ 2024-05-26 03:50 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-26 03:50 UTC (permalink / raw)
To: Marcos Pegoraro <[email protected]>; +Cc: pgsql-hackers
On Thu, May 23, 2024 at 04:54:28PM -0300, Marcos Pegoraro wrote:
> • Rename SLRU columns in system view pg_stat_slru (Alvaro Herrera)
>
> The column names accepted by pg_stat_slru_rest() are also changed.
>
> Is pg_stat_slru_rest() correct ?
Oops, typo, fixed, thanks.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-21 02:20 ` Amit Langote <[email protected]>
2024-05-22 22:46 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Amit Langote @ 2024-05-21 02:20 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, May 9, 2024 at 1:04 PM Bruce Momjian <[email protected]> wrote:
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
> It will be improved until the final release. The item count is 188,
> which is similar to recent releases:
>
> release-10: 189
> release-11: 170
> release-12: 180
> release-13: 178
> release-14: 220
> release-15: 184
> release-16: 206
> release-17: 188
>
> I welcome feedback. For some reason it was an easier job than usual.
Thanks Bruce for working on this as always.
Failed to notice when I read the notes before:
<listitem>
<para>
Add SQL/JSON constructor functions JSON(), JSON_SCALAR(), and
JSON_SERIALIZE() (Amit Langote)
</para>
</listitem>
Should be:
<listitem>
<para>
Add SQL/JSON constructor functions JSON(), JSON_SCALAR(), and
JSON_SERIALIZE() (Nikita Glukhov, Teodor Sigaev, Oleg Bartunov,
Alexander Korotkov, Andrew Dunstan, Amit Langote)
</para>
</listitem>
Patch attached.
--
Thanks, Amit Langote
Attachments:
[application/octet-stream] sql-json-credits.patch (565B, ../../CA+HiwqHA2_2V-UtdEEjX3wMUcO=pAwH2D=9P9cRYGVcNLJkH+w@mail.gmail.com/2-sql-json-credits.patch)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 0e7ff51f4a..c5aa7d9c25 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -1608,7 +1608,7 @@ Author: Amit Langote <[email protected]>
<listitem>
<para>
-Add SQL/JSON constructor functions JSON(), JSON_SCALAR(), and JSON_SERIALIZE() (Amit Langote)
+Add SQL/JSON constructor functions JSON(), JSON_SCALAR(), and JSON_SERIALIZE() (Nikita Glukhov, Teodor Sigaev, Oleg Bartunov, Alexander Korotkov, Andrew Dunstan, Amit Langote)
</para>
</listitem>
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 02:20 ` Re: First draft of PG 17 release notes Amit Langote <[email protected]>
@ 2024-05-22 22:46 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-22 22:46 UTC (permalink / raw)
To: Amit Langote <[email protected]>; +Cc: pgsql-hackers
On Tue, May 21, 2024 at 11:20:02AM +0900, Amit Langote wrote:
> Thanks Bruce for working on this as always.
>
> Failed to notice when I read the notes before:
>
> <listitem>
> <para>
> Add SQL/JSON constructor functions JSON(), JSON_SCALAR(), and
> JSON_SERIALIZE() (Amit Langote)
> </para>
> </listitem>
>
> Should be:
>
> <listitem>
> <para>
> Add SQL/JSON constructor functions JSON(), JSON_SCALAR(), and
> JSON_SERIALIZE() (Nikita Glukhov, Teodor Sigaev, Oleg Bartunov,
> Alexander Korotkov, Andrew Dunstan, Amit Langote)
> </para>
> </listitem>
Thanks, applied.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-22 02:29 ` Masahiko Sawada <[email protected]>
2024-05-22 22:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Masahiko Sawada @ 2024-05-22 02:29 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
Hi,
On Thu, May 9, 2024 at 1:03 PM Bruce Momjian <[email protected]> wrote:
>
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
> It will be improved until the final release. The item count is 188,
> which is similar to recent releases:
>
I found a typo:
s/pg_statstatement/pg_stat_statement/
I've attached a patch to fix it.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
Attachments:
[application/octet-stream] fix_pg_stat_statements.patch (537B, ../../CAD21AoB_MR=S_Gh=oeJR4ji0GGY+d8747O-5pYcbMbhGOMtAwQ@mail.gmail.com/2-fix_pg_stat_statements.patch)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 612eb100a7..2aecb8ba9b 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -2722,7 +2722,7 @@ This is useful for testing.
</itemizedlist>
<sect4 id="release-17-pgstatstatements">
- <title><link linkend="pgstatstatements"><application>pg_statstatements</application></link></title>
+ <title><link linkend="pgstatstatements"><application>pg_stat_statements</application></link></title>
<itemizedlist>
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-22 02:29 ` Re: First draft of PG 17 release notes Masahiko Sawada <[email protected]>
@ 2024-05-22 22:48 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-22 22:48 UTC (permalink / raw)
To: Masahiko Sawada <[email protected]>; +Cc: pgsql-hackers
On Wed, May 22, 2024 at 11:29:06AM +0900, Masahiko Sawada wrote:
> I found a typo:
>
> s/pg_statstatement/pg_stat_statement/
>
> I've attached a patch to fix it.
Agreed, applied, thanks.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-22 12:25 ` torikoshia <[email protected]>
2024-05-22 22:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: torikoshia @ 2024-05-22 12:25 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On 2024-05-09 13:03, Bruce Momjian wrote:
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
>
> It will be improved until the final release. The item count is 188,
> which is similar to recent releases:
>
> release-10: 189
> release-11: 170
> release-12: 180
> release-13: 178
> release-14: 220
> release-15: 184
> release-16: 206
> release-17: 188
>
> I welcome feedback. For some reason it was an easier job than usual.
Thanks for working on this as always.
<para>
Allow pg_stat_reset_shared("slru") to clear SLRU statistics (Atsushi
Torikoshi)
</para>
Considering someone may copy and paste this, 'slru' is better than
"slru", isn't it?
I also found an older release note[1] used single quotes for this like:
Add pg_stat_reset_shared('bgwriter') to reset the cluster-wide shared
statistics for the background writer (Greg Smith)
[1] https://www.postgresql.org/docs/release/9.0.0/
--
Regards,
--
Atsushi Torikoshi
NTT DATA Group Corporation
Attachments:
[text/x-diff] fix_pg_stat_reset_shared_quotation.patch (448B, ../../[email protected]/2-fix_pg_stat_reset_shared_quotation.patch)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 612eb100a7..96be7b3e8b 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -647,7 +647,7 @@ Author: Michael Paquier <[email protected]>
<listitem>
<para>
-Allow pg_stat_reset_shared("slru") to clear SLRU statistics (Atsushi Torikoshi)
+Allow pg_stat_reset_shared('slru') to clear SLRU statistics (Atsushi Torikoshi)
</para>
<para>
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-22 12:25 ` Re: First draft of PG 17 release notes torikoshia <[email protected]>
@ 2024-05-22 22:50 ` Bruce Momjian <[email protected]>
2024-05-24 00:19 ` Re: First draft of PG 17 release notes Peter Geoghegan <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-22 22:50 UTC (permalink / raw)
To: torikoshia <[email protected]>; +Cc: pgsql-hackers
On Wed, May 22, 2024 at 09:25:41PM +0900, torikoshia wrote:
> Thanks for working on this as always.
>
> <para>
> Allow pg_stat_reset_shared("slru") to clear SLRU statistics (Atsushi
> Torikoshi)
> </para>
>
> Considering someone may copy and paste this, 'slru' is better than "slru",
> isn't it?
> I also found an older release note[1] used single quotes for this like:
>
> Add pg_stat_reset_shared('bgwriter') to reset the cluster-wide shared
> statistics for the background writer (Greg Smith)
Agreed, patch applied, thanks.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-22 12:25 ` Re: First draft of PG 17 release notes torikoshia <[email protected]>
2024-05-22 22:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-24 00:19 ` Peter Geoghegan <[email protected]>
2024-05-26 03:57 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Peter Geoghegan @ 2024-05-24 00:19 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: torikoshia <[email protected]>; pgsql-hackers
On Wed, May 22, 2024 at 6:50 PM Bruce Momjian <[email protected]> wrote:
> Agreed, patch applied, thanks.
The item for my commit 5bf748b8 currently reads:
"Allow btree indexes to more efficiently find array matches"
I think that this isn't likely to mean much to most users. It seems
like it'd be a lot clearer if the wording was more in line with the
beta1 announcement, which talks about the work as an enhancement to
index scans that use an IN ( ) condition. Specifically referencing
IN() (as opposed to something about arrays or array conditions) is
likely to make the item much more understandable.
Referencing array matches makes me think of a GIN index on an array
column. While ScalarArrayOps do use an array under the cover, that's
mostly an implementation detail. At least it is to users that
exclusively use IN(), likely the majority (that's the SQL standard
syntax).
For context, the Postgres 9.2 release notes described the feature that
my work directly built on as follows:
"Allow indexed_col op ANY(ARRAY[...]) conditions to be used in plain
index scans and index-only scans"
This was a very accurate description of this earlier work. Similar
wording could be used now, but that doesn't seem great to me either.
Simply because this wording also doesn't reference IN() conditions in
index quals.
--
Peter Geoghegan
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-22 12:25 ` Re: First draft of PG 17 release notes torikoshia <[email protected]>
2024-05-22 22:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 00:19 ` Re: First draft of PG 17 release notes Peter Geoghegan <[email protected]>
@ 2024-05-26 03:57 ` Bruce Momjian <[email protected]>
2024-05-28 02:44 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-26 03:57 UTC (permalink / raw)
To: Peter Geoghegan <[email protected]>; +Cc: torikoshia <[email protected]>; pgsql-hackers
On Thu, May 23, 2024 at 08:19:15PM -0400, Peter Geoghegan wrote:
> On Wed, May 22, 2024 at 6:50 PM Bruce Momjian <[email protected]> wrote:
> > Agreed, patch applied, thanks.
>
> The item for my commit 5bf748b8 currently reads:
>
> "Allow btree indexes to more efficiently find array matches"
>
> I think that this isn't likely to mean much to most users. It seems
> like it'd be a lot clearer if the wording was more in line with the
> beta1 announcement, which talks about the work as an enhancement to
> index scans that use an IN ( ) condition. Specifically referencing
> IN() (as opposed to something about arrays or array conditions) is
> likely to make the item much more understandable.
>
> Referencing array matches makes me think of a GIN index on an array
> column. While ScalarArrayOps do use an array under the cover, that's
> mostly an implementation detail. At least it is to users that
> exclusively use IN(), likely the majority (that's the SQL standard
> syntax).
>
> For context, the Postgres 9.2 release notes described the feature that
> my work directly built on as follows:
>
> "Allow indexed_col op ANY(ARRAY[...]) conditions to be used in plain
> index scans and index-only scans"
>
> This was a very accurate description of this earlier work. Similar
> wording could be used now, but that doesn't seem great to me either.
> Simply because this wording also doesn't reference IN() conditions in
> index quals.
Agreed. I changed it to:
Allow btree indexes to more efficiently find a set of values, such as
those supplied by IN subqueries
Is that good?
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-22 12:25 ` Re: First draft of PG 17 release notes torikoshia <[email protected]>
2024-05-22 22:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 00:19 ` Re: First draft of PG 17 release notes Peter Geoghegan <[email protected]>
2024-05-26 03:57 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-28 02:44 ` David Rowley <[email protected]>
2024-05-28 04:20 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: David Rowley @ 2024-05-28 02:44 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Peter Geoghegan <[email protected]>; torikoshia <[email protected]>; pgsql-hackers
On Sun, 26 May 2024 at 15:57, Bruce Momjian <[email protected]> wrote:
> Agreed. I changed it to:
>
> Allow btree indexes to more efficiently find a set of values, such as
> those supplied by IN subqueries
>
> Is that good?
I think this needs further adjustment. An "IN subquery" is an IN
clause which contains a subquery. As far as I understand it,
5bf748b86 does nothing to improve those. It's there to improve IN with
a set of values such as IN(1,2,3).
Maybe "IN subqueries" can be replaced with "an SQL IN clause".
David
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-22 12:25 ` Re: First draft of PG 17 release notes torikoshia <[email protected]>
2024-05-22 22:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 00:19 ` Re: First draft of PG 17 release notes Peter Geoghegan <[email protected]>
2024-05-26 03:57 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-28 02:44 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
@ 2024-05-28 04:20 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-28 04:20 UTC (permalink / raw)
To: David Rowley <[email protected]>; +Cc: Peter Geoghegan <[email protected]>; torikoshia <[email protected]>; pgsql-hackers
On Tue, May 28, 2024 at 02:44:28PM +1200, David Rowley wrote:
> On Sun, 26 May 2024 at 15:57, Bruce Momjian <[email protected]> wrote:
> > Agreed. I changed it to:
> >
> > Allow btree indexes to more efficiently find a set of values, such as
> > those supplied by IN subqueries
> >
> > Is that good?
>
> I think this needs further adjustment. An "IN subquery" is an IN
> clause which contains a subquery. As far as I understand it,
> 5bf748b86 does nothing to improve those. It's there to improve IN with
> a set of values such as IN(1,2,3).
>
> Maybe "IN subqueries" can be replaced with "an SQL IN clause".
Okay, I went with:
Allow btree indexes to more efficiently find a set of values,
such as those supplied by IN clauses using constants (Peter Geoghegan,
Matthias van de Meent)
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-23 11:22 ` Alvaro Herrera <[email protected]>
2024-05-26 03:49 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Alvaro Herrera @ 2024-05-23 11:22 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
Hello,
Regarding this item
: Allow the SLRU cache sizes to be configured (Andrey Borodin, Dilip Kumar)
:
: The new server variables are commit_timestamp_buffers,
: multixact_member_buffers, multixact_offset_buffers, notify_buffers,
: serializable_buffers, subtransaction_buffers, and transaction_buffers.
I hereby request to be listed as third author of this feature.
Also, I'd like to suggest to make it more verbose, as details might be
useful to users. Mention that scalability is improved, because
previously we've suggested to recompile with larger #defines, but to be
cautious because values too high degrade performance. Also mention the
point that some of these grow with shared_buffers is user-visible enough
that it warrants an explicit mention. How about like this:
: Allow the SLRU cache sizes to be configured and improve performance of
: larger caches
: (Andrey Borodin, Dilip Kumar, Álvaro Herrera)
:
: The new server variables are commit_timestamp_buffers,
: multixact_member_buffers, multixact_offset_buffers, notify_buffers,
: serializable_buffers, subtransaction_buffers, and transaction_buffers.
: commit_timestamp_buffers, transaction_buffers and
: subtransaction_buffers scale up automatically with shared_buffers.
These three items
: Allow pg_stat_reset_shared() to reset all shared statistics (Atsushi Torikoshi)
:
: This is done by passing NULL.
:
: Allow pg_stat_reset_shared('slru') to clear SLRU statistics (Atsushi Torikoshi)
:
: Now pg_stat_reset_shared(NULL) also resets SLRU statistics.
:
: Allow pg_stat_reset_slru() to reset all SLRU statistics (Bharath Rupireddy)
:
: The command pg_stat_reset_slru(NULL) already did this.
seem a bit repetitive. (I think the first one is also wrong, because it
says you have to pass NULL, but in reality you can also not give an
argument and it works.) Can we make them a single item? Maybe
something like
: Improve reset routines for shared statistics (Atsushi Torikoshi, Bharath Rupireddy)
:
: Resetting all shared statistics can now be done with
: pg_stat_reset_shared() or pg_stat_reset_shared(NULL), while SLRU
: statistics can now be reset with pg_stat_reset_shared('slru'),
: pg_stat_reset_slru() and pg_stat_reset_slru(NULL).
--
Álvaro Herrera PostgreSQL Developer — https://www.EnterpriseDB.com/
"Find a bug in a program, and fix it, and the program will work today.
Show the program how to find and fix a bug, and the program
will work forever" (Oliver Silfridge)
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 11:22 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
@ 2024-05-26 03:49 ` Bruce Momjian <[email protected]>
2024-05-26 08:10 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Bruce Momjian @ 2024-05-26 03:49 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: pgsql-hackers
On Thu, May 23, 2024 at 01:22:51PM +0200, Álvaro Herrera wrote:
> Hello,
>
> Regarding this item
>
> : Allow the SLRU cache sizes to be configured (Andrey Borodin, Dilip Kumar)
> :
> : The new server variables are commit_timestamp_buffers,
> : multixact_member_buffers, multixact_offset_buffers, notify_buffers,
> : serializable_buffers, subtransaction_buffers, and transaction_buffers.
>
> I hereby request to be listed as third author of this feature.
>
> Also, I'd like to suggest to make it more verbose, as details might be
> useful to users. Mention that scalability is improved, because
> previously we've suggested to recompile with larger #defines, but to be
> cautious because values too high degrade performance. Also mention the
> point that some of these grow with shared_buffers is user-visible enough
> that it warrants an explicit mention. How about like this:
>
> : Allow the SLRU cache sizes to be configured and improve performance of
> : larger caches
> : (Andrey Borodin, Dilip Kumar, Álvaro Herrera)
> :
> : The new server variables are commit_timestamp_buffers,
> : multixact_member_buffers, multixact_offset_buffers, notify_buffers,
> : serializable_buffers, subtransaction_buffers, and transaction_buffers.
> : commit_timestamp_buffers, transaction_buffers and
> : subtransaction_buffers scale up automatically with shared_buffers.
Yes, I like that, patch applied.
> These three items
>
> : Allow pg_stat_reset_shared() to reset all shared statistics (Atsushi Torikoshi)
> :
> : This is done by passing NULL.
> :
> : Allow pg_stat_reset_shared('slru') to clear SLRU statistics (Atsushi Torikoshi)
> :
> : Now pg_stat_reset_shared(NULL) also resets SLRU statistics.
> :
> : Allow pg_stat_reset_slru() to reset all SLRU statistics (Bharath Rupireddy)
> :
> : The command pg_stat_reset_slru(NULL) already did this.
>
> seem a bit repetitive. (I think the first one is also wrong, because it
> says you have to pass NULL, but in reality you can also not give an
> argument and it works.) Can we make them a single item? Maybe
> something like
>
> : Improve reset routines for shared statistics (Atsushi Torikoshi, Bharath Rupireddy)
> :
> : Resetting all shared statistics can now be done with
> : pg_stat_reset_shared() or pg_stat_reset_shared(NULL), while SLRU
> : statistics can now be reset with pg_stat_reset_shared('slru'),
> : pg_stat_reset_slru() and pg_stat_reset_slru(NULL).
Andres already suggested improvement for this, and I posted the applied
patch. Can you see if that is good or can be improved? Thanks.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 11:22 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-26 03:49 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-05-26 08:10 ` Alvaro Herrera <[email protected]>
2024-05-26 12:42 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
0 siblings, 1 reply; 194+ messages in thread
From: Alvaro Herrera @ 2024-05-26 08:10 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On 2024-May-25, Bruce Momjian wrote:
> On Thu, May 23, 2024 at 01:22:51PM +0200, Álvaro Herrera wrote:
> > Can we make them a single item? Maybe something like
> >
> > : Improve reset routines for shared statistics (Atsushi Torikoshi, Bharath Rupireddy)
> > :
> > : Resetting all shared statistics can now be done with
> > : pg_stat_reset_shared() or pg_stat_reset_shared(NULL), while SLRU
> > : statistics can now be reset with pg_stat_reset_shared('slru'),
> > : pg_stat_reset_slru() and pg_stat_reset_slru(NULL).
>
> Andres already suggested improvement for this, and I posted the applied
> patch. Can you see if that is good or can be improved? Thanks.
Yeah, looks good, thanks.
--
Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/
"No es bueno caminar con un hombre muerto"
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 11:22 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-26 03:49 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-26 08:10 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
@ 2024-05-26 12:42 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-05-26 12:42 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: pgsql-hackers
On Sun, May 26, 2024 at 10:10:00AM +0200, Álvaro Herrera wrote:
> On 2024-May-25, Bruce Momjian wrote:
>
> > On Thu, May 23, 2024 at 01:22:51PM +0200, Álvaro Herrera wrote:
>
> > > Can we make them a single item? Maybe something like
> > >
> > > : Improve reset routines for shared statistics (Atsushi Torikoshi, Bharath Rupireddy)
> > > :
> > > : Resetting all shared statistics can now be done with
> > > : pg_stat_reset_shared() or pg_stat_reset_shared(NULL), while SLRU
> > > : statistics can now be reset with pg_stat_reset_shared('slru'),
> > > : pg_stat_reset_slru() and pg_stat_reset_slru(NULL).
> >
> > Andres already suggested improvement for this, and I posted the applied
> > patch. Can you see if that is good or can be improved? Thanks.
>
> Yeah, looks good, thanks.
Wow, that's great. My head started to spin trying to make sense of how
those three entries connected. Glad we were able to condense them, and
the new result is easier to read.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
@ 2024-06-05 22:46 ` Dean Rasheed <[email protected]>
2024-06-06 00:53 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
27 siblings, 1 reply; 194+ messages in thread
From: Dean Rasheed @ 2024-06-05 22:46 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On Thu, 9 May 2024 at 05:03, Bruce Momjian <[email protected]> wrote:
>
> I have committed the first draft of the PG 17 release notes; you can
> see the results here:
>
> https://momjian.us/pgsql_docs/release-17.html
I noticed a couple more things. This item:
Add functions to convert integers to hex and binary strings
should read:
Add functions to convert integers to binary and octal strings
The "Improve psql tab completion" item should include this commit:
Author: Michael Paquier <[email protected]>
2024-05-01 [2800fbb2b] Add tab completion for EXPLAIN (MEMORY|SERIALIZE)
and credit Jian He.
Regards,
Dean
^ permalink raw reply [nested|flat] 194+ messages in thread
* Re: First draft of PG 17 release notes
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-06-05 22:46 ` Re: First draft of PG 17 release notes Dean Rasheed <[email protected]>
@ 2024-06-06 00:53 ` Bruce Momjian <[email protected]>
0 siblings, 0 replies; 194+ messages in thread
From: Bruce Momjian @ 2024-06-06 00:53 UTC (permalink / raw)
To: Dean Rasheed <[email protected]>; +Cc: pgsql-hackers
On Wed, Jun 5, 2024 at 11:46:17PM +0100, Dean Rasheed wrote:
> On Thu, 9 May 2024 at 05:03, Bruce Momjian <[email protected]> wrote:
> >
> > I have committed the first draft of the PG 17 release notes; you can
> > see the results here:
> >
> > https://momjian.us/pgsql_docs/release-17.html
>
> I noticed a couple more things. This item:
>
> Add functions to convert integers to hex and binary strings
>
> should read:
>
> Add functions to convert integers to binary and octal strings
>
>
> The "Improve psql tab completion" item should include this commit:
>
> Author: Michael Paquier <[email protected]>
> 2024-05-01 [2800fbb2b] Add tab completion for EXPLAIN (MEMORY|SERIALIZE)
>
> and credit Jian He.
Agreed, attached patch applied.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
Attachments:
[text/x-diff] master.diff (1.2K, ../../[email protected]/2-master.diff)
download | inline diff:
diff --git a/doc/src/sgml/release-17.sgml b/doc/src/sgml/release-17.sgml
index 1131f2aab51..93bc7408b5b 100644
--- a/doc/src/sgml/release-17.sgml
+++ b/doc/src/sgml/release-17.sgml
@@ -1685,7 +1685,7 @@ Author: Nathan Bossart <[email protected]>
<listitem>
<para>
-Add functions to convert integers to hex and binary strings (Eric Radman, Nathan Bossart)
+Add functions to convert integers to binary and octal strings (Eric Radman, Nathan Bossart)
</para>
<para>
@@ -2000,11 +2000,13 @@ Author: Masahiko Sawada <[email protected]>
2024-04-08 [304b6b1a6] Add more tab completion support for ALTER DEFAULT PRIVIL
Author: Alexander Korotkov <[email protected]>
2024-04-30 [60ae37a8b] Add tab completion for partition MERGE/SPLIT operations
+Author: Michael Paquier <[email protected]>
+2024-05-01 [2800fbb2b] Add tab completion for EXPLAIN (MEMORY|SERIALIZE)
-->
<listitem>
<para>
-Improve psql tab completion (Dagfinn Ilmari Mannsåker, Gilles Darold, Christoph Heiss, Steve Chavez, Vignesh C, Pavel Borisov)
+Improve psql tab completion (Dagfinn Ilmari Mannsåker, Gilles Darold, Christoph Heiss, Steve Chavez, Vignesh C, Pavel Borisov, Jian He)
</para>
</listitem>
^ permalink raw reply [nested|flat] 194+ messages in thread
end of thread, other threads:[~2024-06-06 00:53 UTC | newest]
Thread overview: 194+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2021-01-10 19:30 [PATCH] Set PD_ALL_VISIBLE and visibility map bits in COPY FREEZE Tomas Vondra <[email protected]>
2024-05-09 04:03 First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 04:44 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-09 04:47 ` Re: First draft of PG 17 release notes Muhammad Ikram <[email protected]>
2024-05-09 04:52 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-09 13:11 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 13:10 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 13:08 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 04:53 ` Re: First draft of PG 17 release notes Bertrand Drouvot <[email protected]>
2024-05-09 13:29 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 05:17 ` Re: First draft of PG 17 release notes Masahiko Sawada <[email protected]>
2024-05-09 13:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 01:10 ` Re: First draft of PG 17 release notes Masahiko Sawada <[email protected]>
2024-05-15 02:20 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 06:37 ` Re: First draft of PG 17 release notes Richard Guo <[email protected]>
2024-05-09 14:17 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 09:18 ` Re: First draft of PG 17 release notes Aleksander Alekseev <[email protected]>
2024-05-09 14:20 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:00 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 14:49 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 18:40 ` Re: First draft of PG 17 release notes Álvaro Herrera <[email protected]>
2024-05-09 23:54 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 02:22 ` Re: First draft of PG 17 release notes Tender Wang <[email protected]>
2024-05-15 02:02 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:22 ` Re: First draft of PG 17 release notes Dagfinn Ilmari Mannsåker <[email protected]>
2024-05-09 10:31 ` Re: First draft of PG 17 release notes Dagfinn Ilmari Mannsåker <[email protected]>
2024-05-09 14:51 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 14:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 10:53 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 10:57 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 15:09 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 11:49 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 15:12 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 15:26 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-09 15:41 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 15:08 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 16:10 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
2024-05-09 16:29 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-09 20:05 ` Re: First draft of PG 17 release notes Thomas Munro <[email protected]>
2024-05-09 20:35 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 08:24 ` Re: First draft of PG 17 release notes Bharath Rupireddy <[email protected]>
2024-05-10 13:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-13 07:16 ` Re: First draft of PG 17 release notes Bharath Rupireddy <[email protected]>
2024-05-10 16:29 ` Re: First draft of PG 17 release notes Daniel Verite <[email protected]>
2024-05-10 19:47 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 20:58 ` Re: First draft of PG 17 release notes Maiquel Grassi <[email protected]>
2024-05-10 16:50 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-10 21:21 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-10 21:31 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
2024-05-10 21:37 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-11 13:57 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-11 14:24 ` Re: First draft of PG 17 release notes Joe Conway <[email protected]>
2024-05-14 00:56 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 12:20 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-15 00:47 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 00:30 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-11 05:27 ` Re: First draft of PG 17 release notes Andy Fan <[email protected]>
2024-05-11 05:57 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-14 00:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 02:32 ` Re: First draft of PG 17 release notes Andy Fan <[email protected]>
2024-05-15 00:37 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-11 19:32 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
2024-05-15 00:39 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 15:50 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
2024-05-18 16:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-18 20:37 ` Re: First draft of PG 17 release notes Andrew Dunstan <[email protected]>
2024-05-14 10:34 ` Re: First draft of PG 17 release notes Elena Indrupskaya <[email protected]>
2024-05-15 00:43 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 13:58 ` Re: First draft of PG 17 release notes Pantelis Theodosiou <[email protected]>
2024-05-15 00:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-14 19:39 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-15 01:00 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 02:03 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-15 02:06 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-15 02:24 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Amit Kapila <[email protected]>
2024-05-15 08:38 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-15 11:17 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-15 13:13 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-16 02:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 03:35 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-18 14:40 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-19 03:53 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-20 00:12 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 06:23 ` Re: First draft of PG 17 release notes John Naylor <[email protected]>
2024-05-20 13:37 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 18:47 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-22 21:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 03:48 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-16 09:49 ` Re: First draft of PG 17 release notes Jelte Fennema-Nio <[email protected]>
2024-05-16 12:09 ` Re: First draft of PG 17 release notes Joe Conway <[email protected]>
2024-05-16 13:09 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-18 15:13 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 18:35 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-20 18:40 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:40 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-22 22:33 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 18:23 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-26 03:41 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 14:55 ` Re: First draft of PG 17 release notes Peter Geoghegan <[email protected]>
2024-05-18 14:59 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 16:27 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-21 16:51 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-21 16:55 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-22 22:05 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 17:06 ` Re: First draft of PG 17 release notes Peter Geoghegan <[email protected]>
2024-05-21 17:50 ` Re: First draft of PG 17 release notes Robert Haas <[email protected]>
2024-05-21 18:26 ` Re: First draft of PG 17 release notes Melanie Plageman <[email protected]>
2024-05-21 18:53 ` Re: First draft of PG 17 release notes Robert Haas <[email protected]>
2024-05-22 22:15 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-22 22:13 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-22 22:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 01:34 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-23 02:01 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 02:27 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-24 03:04 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 03:11 ` Re: First draft of PG 17 release notes Tom Lane <[email protected]>
2024-05-24 03:27 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 17:50 ` Re: First draft of PG 17 release notes Andres Freund <[email protected]>
2024-05-24 18:26 ` Re: First draft of PG 17 release notes Peter Geoghegan <[email protected]>
2024-05-26 02:10 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 13:54 ` Re: First draft of PG 17 release notes Robert Haas <[email protected]>
2024-05-16 02:39 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-16 02:53 ` Re: First draft of PG 17 release notes David G. Johnston <[email protected]>
2024-05-16 02:55 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-16 08:29 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-18 16:11 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-17 13:22 ` Re: First draft of PG 17 release notes jian he <[email protected]>
2024-05-18 21:37 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-17 13:42 ` Re: First draft of PG 17 release notes Daniel Verite <[email protected]>
2024-05-17 20:30 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-18 21:51 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-20 18:48 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-22 22:39 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 15:30 ` Re: First draft of PG 17 release notes Jeff Davis <[email protected]>
2024-05-23 19:54 ` Re: First draft of PG 17 release notes Marcos Pegoraro <[email protected]>
2024-05-26 03:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-21 02:20 ` Re: First draft of PG 17 release notes Amit Langote <[email protected]>
2024-05-22 22:46 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-22 02:29 ` Re: First draft of PG 17 release notes Masahiko Sawada <[email protected]>
2024-05-22 22:48 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-22 12:25 ` Re: First draft of PG 17 release notes torikoshia <[email protected]>
2024-05-22 22:50 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-24 00:19 ` Re: First draft of PG 17 release notes Peter Geoghegan <[email protected]>
2024-05-26 03:57 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-28 02:44 ` Re: First draft of PG 17 release notes David Rowley <[email protected]>
2024-05-28 04:20 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-23 11:22 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-26 03:49 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-05-26 08:10 ` Re: First draft of PG 17 release notes Alvaro Herrera <[email protected]>
2024-05-26 12:42 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
2024-06-05 22:46 ` Re: First draft of PG 17 release notes Dean Rasheed <[email protected]>
2024-06-06 00:53 ` Re: First draft of PG 17 release notes Bruce Momjian <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox