pg.ddx.io pgsql-hackers@postgresql.org mailing list archive
help / color / mirror / Atom feed[PATCH v1 5/6] Handle pg_get_expr default args in system_functions.sql
57+ messages / 11 participants
[nested] [flat]
Commits from this thread:
2026-09-23 5d84c76021 Add VM corruption check to verify_heapam()
2026-08-04 a37e574545 2f768dda53 Silence Coverity warning about unused visibilitymap_clear() result
2026-07-15 5174d157a0 9171f77db2 4d7feebfbd 0672134309 Test that VM clear registers VM buffers
* [PATCH v1 5/6] Handle pg_get_expr default args in system_functions.sql
@ 2025-12-09 19:17 Mark Wong <markwkm@gmail.com>
0 siblings, 0 replies; 57+ messages in thread
From: Mark Wong @ 2025-12-09 19:17 UTC (permalink / raw)
Modernize pg_get_expr to use CREATE OR REPLACE FUNCTION to handle the
optional pretty argument.
---
src/backend/catalog/system_functions.sql | 7 +++++++
src/backend/utils/adt/ruleutils.c | 17 -----------------
src/include/catalog/pg_proc.dat | 5 +----
3 files changed, 8 insertions(+), 21 deletions(-)
inline patch 1: src/backend/catalog/system_functions.sql, src/backend/utils/adt/ruleutils.c, src/include/catalog/pg_proc.dat (+8/-21 lines) download inline-1.patch
diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql
index 1f89dcc7908..81d210a9c45 100644
--- a/src/backend/catalog/system_functions.sql
+++ b/src/backend/catalog/system_functions.sql
@@ -692,6 +692,13 @@ LANGUAGE INTERNAL
PARALLEL SAFE
AS 'pg_get_constraintdef';
+CREATE OR REPLACE FUNCTION
+ pg_get_expr(expr pg_node_tree, relation oid, pretty bool DEFAULT false)
+RETURNS TEXT
+LANGUAGE INTERNAL
+PARALLEL SAFE
+AS 'pg_get_expr';
+
--
-- The default permissions for functions mean that anyone can execute them.
-- A number of functions shouldn't be executable by just anyone, but rather
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index 416144c49f5..9effa02fa2e 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -2573,23 +2573,6 @@ decompile_column_index_array(Datum column_index_array, Oid relId,
*/
Datum
pg_get_expr(PG_FUNCTION_ARGS)
-{
- text *expr = PG_GETARG_TEXT_PP(0);
- Oid relid = PG_GETARG_OID(1);
- text *result;
- int prettyFlags;
-
- prettyFlags = PRETTYFLAG_INDENT;
-
- result = pg_get_expr_worker(expr, relid, prettyFlags);
- if (result)
- PG_RETURN_TEXT_P(result);
- else
- PG_RETURN_NULL();
-}
-
-Datum
-pg_get_expr_ext(PG_FUNCTION_ARGS)
{
text *expr = PG_GETARG_TEXT_PP(0);
Oid relid = PG_GETARG_OID(1);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index a9431ea4037..10c5286bd7d 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -3977,9 +3977,6 @@
{ oid => '1662', descr => 'trigger description',
proname => 'pg_get_triggerdef', provolatile => 's', prorettype => 'text',
proargtypes => 'oid', prosrc => 'pg_get_triggerdef' },
-{ oid => '1716', descr => 'deparse an encoded expression',
- proname => 'pg_get_expr', provolatile => 's', prorettype => 'text',
- proargtypes => 'pg_node_tree oid', prosrc => 'pg_get_expr' },
{ oid => '1665', descr => 'name of sequence for a serial column',
proname => 'pg_get_serial_sequence', provolatile => 's', prorettype => 'text',
proargtypes => 'text text', prosrc => 'pg_get_serial_sequence' },
@@ -8502,7 +8499,7 @@
{ oid => '2509',
descr => 'deparse an encoded expression with pretty-print option',
proname => 'pg_get_expr', provolatile => 's', prorettype => 'text',
- proargtypes => 'pg_node_tree oid bool', prosrc => 'pg_get_expr_ext' },
+ proargtypes => 'pg_node_tree oid bool', prosrc => 'pg_get_expr' },
{ oid => '2510', descr => 'get the prepared statements for this session',
proname => 'pg_prepared_statement', prorows => '1000', proretset => 't',
provolatile => 's', proparallel => 'r', prorettype => 'record',
--
2.51.2
* [PATCH v4 5/6] Handle pg_get_expr default args in system_functions.sql
@ 2025-12-09 19:17 Mark Wong <markwkm@gmail.com>
0 siblings, 0 replies; 57+ messages in thread
From: Mark Wong @ 2025-12-09 19:17 UTC (permalink / raw)
Modernize pg_get_expr to use proargdefaults to handle the optional
pretty argument. That also means any direct function calls now needs to
set the pretty parameter.
---
src/backend/commands/tablecmds.c | 5 +++--
src/backend/utils/adt/ruleutils.c | 17 -----------------
src/include/catalog/pg_proc.dat | 7 +++----
3 files changed, 6 insertions(+), 23 deletions(-)
inline patch 1: src/backend/commands/tablecmds.c, src/backend/utils/adt/ruleutils.c, src/include/catalog/pg_proc.dat (+6/-23 lines) download inline-1.patch
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index b04b0dbd2a0..a523a512949 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -17477,8 +17477,9 @@ decompile_conbin(HeapTuple contup, TupleDesc tupdesc)
if (isnull)
elog(ERROR, "null conbin for constraint %u", con->oid);
- expr = DirectFunctionCall2(pg_get_expr, attr,
- ObjectIdGetDatum(con->conrelid));
+ expr = DirectFunctionCall3(pg_get_expr, attr,
+ ObjectIdGetDatum(con->conrelid),
+ BoolGetDatum(false));
return TextDatumGetCString(expr);
}
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index 03550f0d80b..a70807e84ca 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -2574,23 +2574,6 @@ decompile_column_index_array(Datum column_index_array, Oid relId,
*/
Datum
pg_get_expr(PG_FUNCTION_ARGS)
-{
- text *expr = PG_GETARG_TEXT_PP(0);
- Oid relid = PG_GETARG_OID(1);
- text *result;
- int prettyFlags;
-
- prettyFlags = PRETTYFLAG_INDENT;
-
- result = pg_get_expr_worker(expr, relid, prettyFlags);
- if (result)
- PG_RETURN_TEXT_P(result);
- else
- PG_RETURN_NULL();
-}
-
-Datum
-pg_get_expr_ext(PG_FUNCTION_ARGS)
{
text *expr = PG_GETARG_TEXT_PP(0);
Oid relid = PG_GETARG_OID(1);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 1d1b24a8f4c..69c1e5bb264 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -3987,9 +3987,6 @@
{ oid => '1662', descr => 'trigger description',
proname => 'pg_get_triggerdef', provolatile => 's', prorettype => 'text',
proargtypes => 'oid', prosrc => 'pg_get_triggerdef' },
-{ oid => '1716', descr => 'deparse an encoded expression',
- proname => 'pg_get_expr', provolatile => 's', prorettype => 'text',
- proargtypes => 'pg_node_tree oid', prosrc => 'pg_get_expr' },
{ oid => '1665', descr => 'name of sequence for a serial column',
proname => 'pg_get_serial_sequence', provolatile => 's', prorettype => 'text',
proargtypes => 'text text', prosrc => 'pg_get_serial_sequence' },
@@ -8531,7 +8528,9 @@
{ oid => '2509',
descr => 'deparse an encoded expression with pretty-print option',
proname => 'pg_get_expr', provolatile => 's', prorettype => 'text',
- proargtypes => 'pg_node_tree oid bool', prosrc => 'pg_get_expr_ext' },
+ proargtypes => 'pg_node_tree oid bool',
+ proargnames => '{expr,relation,pretty}', proargdefaults => '{false}',
+ prosrc => 'pg_get_expr' },
{ oid => '2510', descr => 'get the prepared statements for this session',
proname => 'pg_prepared_statement', prorows => '1000', proretset => 't',
provolatile => 's', proparallel => 'r', prorettype => 'record',
--
2.43.0
* [PATCH v5 5/6] Handle pg_get_expr default args in system_functions.sql
@ 2025-12-09 19:17 Mark Wong <markwkm@gmail.com>
0 siblings, 0 replies; 57+ messages in thread
From: Mark Wong @ 2025-12-09 19:17 UTC (permalink / raw)
Modernize pg_get_expr to use proargdefaults to handle the optional
pretty argument. That also means any direct function calls now needs to
set the pretty parameter.
---
src/backend/commands/tablecmds.c | 5 +++--
src/backend/utils/adt/ruleutils.c | 17 -----------------
src/include/catalog/pg_proc.dat | 7 +++----
3 files changed, 6 insertions(+), 23 deletions(-)
inline patch 1: src/backend/commands/tablecmds.c, src/backend/utils/adt/ruleutils.c, src/include/catalog/pg_proc.dat (+6/-23 lines) download inline-1.patch
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 0ce2e81f9c2..fad4202b1ff 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -17694,8 +17694,9 @@ decompile_conbin(HeapTuple contup, TupleDesc tupdesc)
if (isnull)
elog(ERROR, "null conbin for constraint %u", con->oid);
- expr = DirectFunctionCall2(pg_get_expr, attr,
- ObjectIdGetDatum(con->conrelid));
+ expr = DirectFunctionCall3(pg_get_expr, attr,
+ ObjectIdGetDatum(con->conrelid),
+ BoolGetDatum(false));
return TextDatumGetCString(expr);
}
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index 1afa2531f30..972f89eafbc 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -2932,23 +2932,6 @@ decompile_column_index_array(Datum column_index_array, Oid relId,
*/
Datum
pg_get_expr(PG_FUNCTION_ARGS)
-{
- text *expr = PG_GETARG_TEXT_PP(0);
- Oid relid = PG_GETARG_OID(1);
- text *result;
- int prettyFlags;
-
- prettyFlags = PRETTYFLAG_INDENT;
-
- result = pg_get_expr_worker(expr, relid, prettyFlags);
- if (result)
- PG_RETURN_TEXT_P(result);
- else
- PG_RETURN_NULL();
-}
-
-Datum
-pg_get_expr_ext(PG_FUNCTION_ARGS)
{
text *expr = PG_GETARG_TEXT_PP(0);
Oid relid = PG_GETARG_OID(1);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index e148b1ad8e3..f36f4625e50 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -4000,9 +4000,6 @@
{ oid => '1662', descr => 'trigger description',
proname => 'pg_get_triggerdef', provolatile => 's', prorettype => 'text',
proargtypes => 'oid', prosrc => 'pg_get_triggerdef' },
-{ oid => '1716', descr => 'deparse an encoded expression',
- proname => 'pg_get_expr', provolatile => 's', prorettype => 'text',
- proargtypes => 'pg_node_tree oid', prosrc => 'pg_get_expr' },
{ oid => '1665', descr => 'name of sequence for a serial column',
proname => 'pg_get_serial_sequence', provolatile => 's', prorettype => 'text',
proargtypes => 'text text', prosrc => 'pg_get_serial_sequence' },
@@ -8595,7 +8592,9 @@
{ oid => '2509',
descr => 'deparse an encoded expression with pretty-print option',
proname => 'pg_get_expr', provolatile => 's', prorettype => 'text',
- proargtypes => 'pg_node_tree oid bool', prosrc => 'pg_get_expr_ext' },
+ proargtypes => 'pg_node_tree oid bool',
+ proargnames => '{expr,relation,pretty}', proargdefaults => '{false}',
+ prosrc => 'pg_get_expr' },
{ oid => '2510', descr => 'get the prepared statements for this session',
proname => 'pg_prepared_statement', prorows => '1000', proretset => 't',
provolatile => 's', proparallel => 'r', prorettype => 'record',
--
2.52.0
* [PATCH v7 5/6] Handle pg_get_expr default args in system_functions.sql
@ 2025-12-09 19:17 Mark Wong <markwkm@gmail.com>
0 siblings, 0 replies; 57+ messages in thread
From: Mark Wong @ 2025-12-09 19:17 UTC (permalink / raw)
Modernize pg_get_expr to use proargdefaults to handle the optional
pretty argument. That also means any direct function calls now needs to
set the pretty parameter.
---
src/backend/commands/tablecmds.c | 5 +++--
src/backend/utils/adt/ruleutils.c | 17 -----------------
src/include/catalog/pg_proc.dat | 9 ++++-----
3 files changed, 7 insertions(+), 24 deletions(-)
inline patch 1: src/backend/commands/tablecmds.c, src/backend/utils/adt/ruleutils.c, src/include/catalog/pg_proc.dat (+7/-24 lines) download inline-1.patch
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index eec09ba1ded..73eb6322daf 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -17695,8 +17695,9 @@ decompile_conbin(HeapTuple contup, TupleDesc tupdesc)
if (isnull)
elog(ERROR, "null conbin for constraint %u", con->oid);
- expr = DirectFunctionCall2(pg_get_expr, attr,
- ObjectIdGetDatum(con->conrelid));
+ expr = DirectFunctionCall3(pg_get_expr, attr,
+ ObjectIdGetDatum(con->conrelid),
+ BoolGetDatum(false));
return TextDatumGetCString(expr);
}
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index 2d39bce7fd7..707f83d4310 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -2932,23 +2932,6 @@ decompile_column_index_array(Datum column_index_array, Oid relId,
*/
Datum
pg_get_expr(PG_FUNCTION_ARGS)
-{
- text *expr = PG_GETARG_TEXT_PP(0);
- Oid relid = PG_GETARG_OID(1);
- text *result;
- int prettyFlags;
-
- prettyFlags = PRETTYFLAG_INDENT;
-
- result = pg_get_expr_worker(expr, relid, prettyFlags);
- if (result)
- PG_RETURN_TEXT_P(result);
- else
- PG_RETURN_NULL();
-}
-
-Datum
-pg_get_expr_ext(PG_FUNCTION_ARGS)
{
text *expr = PG_GETARG_TEXT_PP(0);
Oid relid = PG_GETARG_OID(1);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 39139fd9e3b..855529509de 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -4006,9 +4006,6 @@
{ oid => '1662', descr => 'trigger description',
proname => 'pg_get_triggerdef', provolatile => 's', prorettype => 'text',
proargtypes => 'oid', prosrc => 'pg_get_triggerdef' },
-{ oid => '1716', descr => 'deparse an encoded expression',
- proname => 'pg_get_expr', provolatile => 's', prorettype => 'text',
- proargtypes => 'pg_node_tree oid', prosrc => 'pg_get_expr' },
{ oid => '1665', descr => 'name of sequence for a serial column',
proname => 'pg_get_serial_sequence', provolatile => 's', prorettype => 'text',
proargtypes => 'text text', prosrc => 'pg_get_serial_sequence' },
@@ -8638,9 +8635,11 @@
pronargdefaults => '1', proargdefaults => '{NULL}',
prosrc => 'pg_get_database_ddl' },
{ oid => '2509',
- descr => 'deparse an encoded expression with pretty-print option',
+ descr => 'deparse an encoded expression',
proname => 'pg_get_expr', provolatile => 's', prorettype => 'text',
- proargtypes => 'pg_node_tree oid bool', prosrc => 'pg_get_expr_ext' },
+ proargtypes => 'pg_node_tree oid bool',
+ proargnames => '{expr,relation,pretty}', proargdefaults => '{false}',
+ prosrc => 'pg_get_expr' },
{ oid => '2510', descr => 'get the prepared statements for this session',
proname => 'pg_prepared_statement', prorows => '1000', proretset => 't',
provolatile => 's', proparallel => 'r', prorettype => 'record',
--
2.52.0
* [PATCH v8 5/6] Handle pg_get_expr default args in system_functions.sql
@ 2025-12-09 19:17 Mark Wong <markwkm@gmail.com>
0 siblings, 0 replies; 57+ messages in thread
From: Mark Wong @ 2025-12-09 19:17 UTC (permalink / raw)
Modernize pg_get_expr to use proargdefaults to handle the optional
pretty argument. That also means any direct function calls now needs to
set the pretty parameter.
---
src/backend/commands/tablecmds.c | 5 +++--
src/backend/utils/adt/ruleutils.c | 17 -----------------
src/include/catalog/pg_proc.dat | 9 ++++-----
3 files changed, 7 insertions(+), 24 deletions(-)
inline patch 1: src/backend/commands/tablecmds.c, src/backend/utils/adt/ruleutils.c, src/include/catalog/pg_proc.dat (+7/-24 lines) download inline-1.patch
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index a1845240a98..f063f744f6d 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -17730,8 +17730,9 @@ decompile_conbin(HeapTuple contup, TupleDesc tupdesc)
if (isnull)
elog(ERROR, "null conbin for constraint %u", con->oid);
- expr = DirectFunctionCall2(pg_get_expr, attr,
- ObjectIdGetDatum(con->conrelid));
+ expr = DirectFunctionCall3(pg_get_expr, attr,
+ ObjectIdGetDatum(con->conrelid),
+ BoolGetDatum(false));
return TextDatumGetCString(expr);
}
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index d83be8cf77d..e60478677e0 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -2932,23 +2932,6 @@ decompile_column_index_array(Datum column_index_array, Oid relId,
*/
Datum
pg_get_expr(PG_FUNCTION_ARGS)
-{
- text *expr = PG_GETARG_TEXT_PP(0);
- Oid relid = PG_GETARG_OID(1);
- text *result;
- int prettyFlags;
-
- prettyFlags = PRETTYFLAG_INDENT;
-
- result = pg_get_expr_worker(expr, relid, prettyFlags);
- if (result)
- PG_RETURN_TEXT_P(result);
- else
- PG_RETURN_NULL();
-}
-
-Datum
-pg_get_expr_ext(PG_FUNCTION_ARGS)
{
text *expr = PG_GETARG_TEXT_PP(0);
Oid relid = PG_GETARG_OID(1);
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 7e452aa8c27..71bd2aaf3d9 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -4006,9 +4006,6 @@
{ oid => '1662', descr => 'trigger description',
proname => 'pg_get_triggerdef', provolatile => 's', prorettype => 'text',
proargtypes => 'oid', prosrc => 'pg_get_triggerdef' },
-{ oid => '1716', descr => 'deparse an encoded expression',
- proname => 'pg_get_expr', provolatile => 's', prorettype => 'text',
- proargtypes => 'pg_node_tree oid', prosrc => 'pg_get_expr' },
{ oid => '1665', descr => 'name of sequence for a serial column',
proname => 'pg_get_serial_sequence', provolatile => 's', prorettype => 'text',
proargtypes => 'text text', prosrc => 'pg_get_serial_sequence' },
@@ -8605,9 +8602,11 @@
proargmodes => '{i,v}', proargdefaults => '{NULL}',
prosrc => 'pg_get_database_ddl' },
{ oid => '2509',
- descr => 'deparse an encoded expression with pretty-print option',
+ descr => 'deparse an encoded expression',
proname => 'pg_get_expr', provolatile => 's', prorettype => 'text',
- proargtypes => 'pg_node_tree oid bool', prosrc => 'pg_get_expr_ext' },
+ proargtypes => 'pg_node_tree oid bool',
+ proargnames => '{expr,relation,pretty}', proargdefaults => '{false}',
+ prosrc => 'pg_get_expr' },
{ oid => '2510', descr => 'get the prepared statements for this session',
proname => 'pg_prepared_statement', prorows => '1000', proretset => 't',
provolatile => 's', proparallel => 'r', prorettype => 'record',
--
2.53.0
* Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-05 19:56 Melanie Plageman <melanieplageman@gmail.com>
0 siblings, 2 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-03-05 19:56 UTC (permalink / raw)
To: PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; +Cc: Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>; Andres Freund <andres@anarazel.de>
Today Andres and I noticed that heap_{update,insert,delete}() don't
register the VM buffer when they are clearing the VM. I was under the
impression that any buffers modified needed to be registered in the
WAL record. Without which, you'll never do an FPI. It seems like this
could cause checksum failures. What are we missing?
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-05 20:16 Andres Freund <andres@anarazel.de>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 0 replies; 57+ messages in thread
From: Andres Freund @ 2026-03-05 20:16 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>; Noah Misch <noah@leadboat.com>
Hi,
On 2026-03-05 14:56:13 -0500, Melanie Plageman wrote:
> Today Andres and I noticed that heap_{update,insert,delete}() don't
> register the VM buffer when they are clearing the VM. I was under the
> impression that any buffers modified needed to be registered in the
> WAL record. Without which, you'll never do an FPI. It seems like this
> could cause checksum failures. What are we missing?
Besides checksum errors, I was also worried that this would break
pg_rewind. But it seems we're saved by pg_rewind only parsing the WAL for
changes to the main fork.
But it does seem like it could be a problem for incremental backup /
walsummarizer?
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-05 20:23 Matthias van de Meent <boekewurm+postgres@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 1 reply; 57+ messages in thread
From: Matthias van de Meent @ 2026-03-05 20:23 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; Andres Freund <andres@anarazel.de>; +Cc: PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
On Thu, 5 Mar 2026 at 20:56, Melanie Plageman <melanieplageman@gmail.com> wrote:
>
> Today Andres and I noticed that heap_{update,insert,delete}() don't
> register the VM buffer when they are clearing the VM. I was under the
> impression that any buffers modified needed to be registered in the
> WAL record. Without which, you'll never do an FPI. It seems like this
> could cause checksum failures. What are we missing?
Might it be, because zeroing a VM page (which you would do when you
encounter checksum failures) is an MVCC-safe operation?
I agree with you that we probably _should_ register the VM (and
possibly FSM) buffer, but that's a bit of a different story. Right
now, the VM does not use the standard page format (nor does the FSM),
and therefore every FPI would be the full 8KB, even when just a few
bits of the VM page are in use; which would be a rather large waste of
space.
On Thu, 5 Mar 2026 at 21:16, Andres Freund <andres@anarazel.de> wrote:
>
> But it does seem like it could be a problem for incremental backup /
> walsummarizer?
I don't think it is, because that doesn't do calculations for non-main
forks, it considers those forks always changed and includes them in
full. Or at least, that was the response I got when I raised concerns
about the FSM back when the incremental backup feature was being
developed [0].
Kind regards,
Matthias van de Meent
Databricks (https://databricks.com)
[0] https://postgr.es/m/CA%2BTgmoaR8o%2BPBeWc_2Ge0XVgoM7xWKNyDmqXoTov%3DS6_J1gecQ%40mail.gmail.com
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-05 20:38 Andres Freund <andres@anarazel.de>
parent: Matthias van de Meent <boekewurm+postgres@gmail.com>
0 siblings, 2 replies; 57+ messages in thread
From: Andres Freund @ 2026-03-05 20:38 UTC (permalink / raw)
To: Matthias van de Meent <boekewurm+postgres@gmail.com>; +Cc: Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
Hi,
On 2026-03-05 21:23:12 +0100, Matthias van de Meent wrote:
> On Thu, 5 Mar 2026 at 20:56, Melanie Plageman <melanieplageman@gmail.com> wrote:
> >
> > Today Andres and I noticed that heap_{update,insert,delete}() don't
> > register the VM buffer when they are clearing the VM. I was under the
> > impression that any buffers modified needed to be registered in the
> > WAL record. Without which, you'll never do an FPI. It seems like this
> > could cause checksum failures. What are we missing?
>
> Might it be, because zeroing a VM page (which you would do when you
> encounter checksum failures) is an MVCC-safe operation?
We are afaict not automatically zeroing corrupted VM pages (we are doing that
for FSM though). IMO it's a bug if a user ever has to turn on
zero_damaged_pages outside of the storage actually corrupting data.
> I agree with you that we probably _should_ register the VM (and possibly
> FSM) buffer, but that's a bit of a different story. Right now, the VM does
> not use the standard page format (nor does the FSM), and therefore every FPI
> would be the full 8KB, even when just a few bits of the VM page are in use;
> which would be a rather large waste of space.
If checksums are enabled, we are already emitting FPIs for the VM when
*setting* bits in the VM (c.f. log_heap_visible()). I don't see why the story
for clearing it should be different. And because there are so few VM pages
compared to heap pages, I wouldn't expect there to be a meaningful amount of
VM FPIs outside of very contrived workloads.
> On Thu, 5 Mar 2026 at 21:16, Andres Freund <andres@anarazel.de> wrote:
> >
> > But it does seem like it could be a problem for incremental backup /
> > walsummarizer?
>
> I don't think it is, because that doesn't do calculations for non-main
> forks, it considers those forks always changed and includes them in
> full. Or at least, that was the response I got when I raised concerns
> about the FSM back when the incremental backup feature was being
> developed [0].
There's explicit code for ignoring the FSM, but I don't see the same for the
VM. And that makes sense: VM changes are mostly WAL logged, just not
completely / generically (i.e. this complaint), whereas FSM changes are not
WAL logged at all.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-05 21:01 Andres Freund <andres@anarazel.de>
parent: Andres Freund <andres@anarazel.de>
1 sibling, 2 replies; 57+ messages in thread
From: Andres Freund @ 2026-03-05 21:01 UTC (permalink / raw)
To: Matthias van de Meent <boekewurm+postgres@gmail.com>; +Cc: Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
Hi,
On 2026-03-05 15:38:24 -0500, Andres Freund wrote:
> > On Thu, 5 Mar 2026 at 21:16, Andres Freund <andres@anarazel.de> wrote:
> > >
> > > But it does seem like it could be a problem for incremental backup /
> > > walsummarizer?
> >
> > I don't think it is, because that doesn't do calculations for non-main
> > forks, it considers those forks always changed and includes them in
> > full. Or at least, that was the response I got when I raised concerns
> > about the FSM back when the incremental backup feature was being
> > developed [0].
>
> There's explicit code for ignoring the FSM, but I don't see the same for the
> VM. And that makes sense: VM changes are mostly WAL logged, just not
> completely / generically (i.e. this complaint), whereas FSM changes are not
> WAL logged at all.
Unfortunately I can confirm that incremental backups end up with an outdated
VM.
An unfortunate kicker: It looks like verify_heapam() doesn't even notice :(.
But the next vacuum does complain, if we end up processing the page for some
reason:
2026-03-05 15:56:23.018 EST [2073843][client backend][0/5:0][psql] WARNING: page is not marked all-visible but visibility map bit is set in relation "s" page 0
2026-03-05 15:56:23.018 EST [2073843][client backend][0/5:0][psql] CONTEXT: while scanning block 0 of relation "public.s"
Repro:
1) CREATE TABLE s(id int); INSERT INTO s DEFAULT VALUES; SELECT pg_current_wal_insert_lsn(); VACUUM s;
2) create full base backup
rm -rf /tmp/backup-a && mkdir /tmp/backup-a && ./src/bin/pg_basebackup/pg_basebackup -c fast -D /tmp/backup-a
3) UPDATE s SET id = -id;
4) create incremental backup
rm -rf /tmp/backup-a-i && mkdir /tmp/backup-a-i && ./src/bin/pg_basebackup/pg_basebackup --incremental=/tmp/backup-a/backup_manifest -c fast -D /tmp/backup-a-i
5) combine full and incremental backup
rm -rf /tmp/backup-a-i-c && mkdir /tmp/backup-a-i-c && ./src/bin/pg_combinebackup/pg_combinebackup -d -o /tmp/backup-a-i-c --copy-file-range /tmp/backup-a /tmp/backup-a-i/
6) start PG on combined backup and use
SELECT * FROM pg_visibility_map_summary('s');
to see things are wrong
7) vacuum and see it complain (page is processed because we always look at the
last page, and there's just one)
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-05 23:11 Robert Haas <robertmhaas@gmail.com>
parent: Andres Freund <andres@anarazel.de>
1 sibling, 1 reply; 57+ messages in thread
From: Robert Haas @ 2026-03-05 23:11 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; +Cc: Matthias van de Meent <boekewurm+postgres@gmail.com>; Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Thu, Mar 5, 2026 at 3:38 PM Andres Freund <andres@anarazel.de> wrote:
> If checksums are enabled, we are already emitting FPIs for the VM when
> *setting* bits in the VM (c.f. log_heap_visible()). I don't see why the story
> for clearing it should be different. And because there are so few VM pages
> compared to heap pages, I wouldn't expect there to be a meaningful amount of
> VM FPIs outside of very contrived workloads.
Yeah, that's how it seems to me, too, at least as of this moment.
Is there any indication in the code or comments that this was an
intentional omission because somebody thought we could get away
without doing it? Or is just a straight-up goof?
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-06 00:43 Andres Freund <andres@anarazel.de>
parent: Robert Haas <robertmhaas@gmail.com>
0 siblings, 1 reply; 57+ messages in thread
From: Andres Freund @ 2026-03-06 00:43 UTC (permalink / raw)
To: Robert Haas <robertmhaas@gmail.com>; +Cc: Matthias van de Meent <boekewurm+postgres@gmail.com>; Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
Hi,
On 2026-03-05 18:11:43 -0500, Robert Haas wrote:
> On Thu, Mar 5, 2026 at 3:38 PM Andres Freund <andres@anarazel.de> wrote:
> > If checksums are enabled, we are already emitting FPIs for the VM when
> > *setting* bits in the VM (c.f. log_heap_visible()). I don't see why the story
> > for clearing it should be different. And because there are so few VM pages
> > compared to heap pages, I wouldn't expect there to be a meaningful amount of
> > VM FPIs outside of very contrived workloads.
>
> Yeah, that's how it seems to me, too, at least as of this moment.
>
> Is there any indication in the code or comments that this was an
> intentional omission because somebody thought we could get away
> without doing it?
I couldn't find any evidence of that, which of course doesn't mean it doesn't
exist.
> Or is just a straight-up goof?
It kinda looks like a victim of multiple subsequent changes that each arguably
should have done something different:
commit 2c03216d831
Author: Heikki Linnakangas <heikki.linnakangas@iki.fi>
Date: 2014-11-20 17:56:26 +0200
Revamp the WAL record format.
Which should have included a reference to the VM page in the WAL record for
insert/update/delete.
and
commit 96ef3b8ff1c
Author: Simon Riggs <simon@2ndQuadrant.com>
Date: 2013-03-22 13:54:07 +0000
Allow I/O reliability checks using 16-bit checksums
Which should have triggered an FPI when clearing the VM, as afaict there
otherwise is no guarantee the checksum will be correct after a crash.
and, although I am not sure it's true:
commit 503c7305a1e
Author: Robert Haas <rhaas@postgresql.org>
Date: 2011-06-21 23:04:40 -0400
Make the visibility map crash-safe.
Which perhaps also should have emitted an FPI when clearing a bit? But I'm
unsure that that was required at the time. OTOH, it did seem to generate an
FPI for setting a VM bit, so ...
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-06 16:49 Robert Haas <robertmhaas@gmail.com>
parent: Andres Freund <andres@anarazel.de>
0 siblings, 1 reply; 57+ messages in thread
From: Robert Haas @ 2026-03-06 16:49 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; +Cc: Matthias van de Meent <boekewurm+postgres@gmail.com>; Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Thu, Mar 5, 2026 at 7:43 PM Andres Freund <andres@anarazel.de> wrote:
> Which perhaps also should have emitted an FPI when clearing a bit? But I'm
> unsure that that was required at the time. OTOH, it did seem to generate an
> FPI for setting a VM bit, so ...
After booting up the part of my brain that worked on this back in
2011, I think I reasoned that clearing a bit was idempotent and didn't
depend on the prior page state, so that we would be adequate protected
without an FPI. I had to add a WAL record to *set* the VM bit, because
that wasn't logged at all, but the actual of clearing the VM bit
piggybacked on the heap change that necessitated doing so, and having
that record also emit an FPI for the VM page didn't seem to me to add
anything.
So I think technically it's the addition of checksums that turns this
into a real bug, because now torn pages are a checksum-failure hazard.
However, that would have been extremely hard to notice given that I
seem never to have actually documented that reasoning in a comment
anywhere.
I don't think we should mess around with trying to make the behavior
here conditional on wal_level, checksums, etc. We should just do it
all the time to keep the logic simple.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-12 11:25 Yura Sokolov <y.sokolov@postgrespro.ru>
parent: Andres Freund <andres@anarazel.de>
1 sibling, 1 reply; 57+ messages in thread
From: Yura Sokolov @ 2026-03-12 11:25 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; +Cc: Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>; Andrey M. Borodin <x4mmm@yandex-team.ru>
06.03.2026 00:01, Andres Freund пишет:
> Hi,
>
> On 2026-03-05 15:38:24 -0500, Andres Freund wrote:
>>> On Thu, 5 Mar 2026 at 21:16, Andres Freund <andres@anarazel.de> wrote:
>>>>
>>>> But it does seem like it could be a problem for incremental backup /
>>>> walsummarizer?
>>>
>>> I don't think it is, because that doesn't do calculations for non-main
>>> forks, it considers those forks always changed and includes them in
>>> full. Or at least, that was the response I got when I raised concerns
>>> about the FSM back when the incremental backup feature was being
>>> developed [0].
>>
>> There's explicit code for ignoring the FSM, but I don't see the same for the
>> VM. And that makes sense: VM changes are mostly WAL logged, just not
>> completely / generically (i.e. this complaint), whereas FSM changes are not
>> WAL logged at all.
>
> Unfortunately I can confirm that incremental backups end up with an outdated
> VM.
That is why pg_probackup still archive VM at whole in incremental (WAL
parsing) backup.
That is why WAL-G's incremental backup in WAL-parsing mode is (was?)
considered unstable.
I know the problem for couple of years. Excuse me I didn't write about.
I didn't recognize fix could be as simple as registering VM buffers.
My bad. I fill so stupid :-(
It would be great if it will be fixed in all supported versions.
--
regards
Yura Sokolov aka funny-falcon
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-12 12:22 Yura Sokolov <y.sokolov@postgrespro.ru>
parent: Yura Sokolov <y.sokolov@postgrespro.ru>
0 siblings, 0 replies; 57+ messages in thread
From: Yura Sokolov @ 2026-03-12 12:22 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; +Cc: Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>; Andrey M. Borodin <x4mmm@yandex-team.ru>
12.03.2026 14:25, Yura Sokolov пишет:
> 06.03.2026 00:01, Andres Freund пишет:
>> Hi,
>>
>> On 2026-03-05 15:38:24 -0500, Andres Freund wrote:
>>>> On Thu, 5 Mar 2026 at 21:16, Andres Freund <andres@anarazel.de> wrote:
>>>>>
>>>>> But it does seem like it could be a problem for incremental backup /
>>>>> walsummarizer?
>>>>
>>>> I don't think it is, because that doesn't do calculations for non-main
>>>> forks, it considers those forks always changed and includes them in
>>>> full. Or at least, that was the response I got when I raised concerns
>>>> about the FSM back when the incremental backup feature was being
>>>> developed [0].
>>>
>>> There's explicit code for ignoring the FSM, but I don't see the same for the
>>> VM. And that makes sense: VM changes are mostly WAL logged, just not
>>> completely / generically (i.e. this complaint), whereas FSM changes are not
>>> WAL logged at all.
>>
>> Unfortunately I can confirm that incremental backups end up with an outdated
>> VM.
>
> That is why pg_probackup still archive VM at whole in incremental (WAL
> parsing) backup.
> That is why WAL-G's incremental backup in WAL-parsing mode is (was?)
> considered unstable.
>
> I know the problem for couple of years. Excuse me I didn't write about.
>
> I didn't recognize fix could be as simple as registering VM buffers.
> My bad. I fill so stupid :-(
Should LSN properly set as well on vm page? To maximum of concurrent
updates, of course.
--
regards
Yura Sokolov aka funny-falcon
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-12 17:05 Andres Freund <andres@anarazel.de>
parent: Robert Haas <robertmhaas@gmail.com>
0 siblings, 2 replies; 57+ messages in thread
From: Andres Freund @ 2026-03-12 17:05 UTC (permalink / raw)
To: Robert Haas <robertmhaas@gmail.com>; Thomas Munro <thomas.munro@gmail.com>; +Cc: Matthias van de Meent <boekewurm+postgres@gmail.com>; Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
Hi,
On 2026-03-06 11:49:20 -0500, Robert Haas wrote:
> On Thu, Mar 5, 2026 at 7:43 PM Andres Freund <andres@anarazel.de> wrote:
> > Which perhaps also should have emitted an FPI when clearing a bit? But I'm
> > unsure that that was required at the time. OTOH, it did seem to generate an
> > FPI for setting a VM bit, so ...
>
> After booting up the part of my brain that worked on this back in
> 2011, I think I reasoned that clearing a bit was idempotent and didn't
> depend on the prior page state, so that we would be adequate protected
> without an FPI. I had to add a WAL record to *set* the VM bit, because
> that wasn't logged at all, but the actual of clearing the VM bit
> piggybacked on the heap change that necessitated doing so, and having
> that record also emit an FPI for the VM page didn't seem to me to add
> anything.
>
> So I think technically it's the addition of checksums that turns this
> into a real bug, because now torn pages are a checksum-failure hazard.
> However, that would have been extremely hard to notice given that I
> seem never to have actually documented that reasoning in a comment
> anywhere.
And then it got a lot worse with incremental backup support...
> I don't think we should mess around with trying to make the behavior
> here conditional on wal_level, checksums, etc. We should just do it
> all the time to keep the logic simple.
Agreed.
I've been working on a fix for this (still pretty raw though). What a mess.
As part of validating that I added error checks that disallow reads in the
startup process for anything other than the FSM fork. Which triggered, due to
visibilitymap_prepare_truncate() reading a VM page.
Which in turn made me wonder: Is it actually OK for
visibilitymap_prepare_truncate() to do separate WAL logging from
XLOG_SMGR_TRUNCATE?
I suspect not, consider this scenario:
1) primary does XLOG_FPI for the VM page
2) checkpoint
3) primary does XLOG_SMGR_TRUNCATE
4) standby replays XLOG_FPI
5) primary performs a restartpoint
6) primary replays XLOG_SMGR_TRUNCATE, as part of that it again does
visibilitymap_prepare_truncate, which dirties the VM page
7) standby crashes while writing out the VM page
8) standby does recovery and now finds a torn VM page, triggering a checksum
failure
Perhaps that's kinda ok, because vm_readbuf() uses ZERO_ON_ERROR, so we'd just
wipe out that region of the VM?
Separately, is it actually sufficient that visibilitymap_prepare_truncate()
only WAL logs the changes if XLogHintBitIsNeeded()? I'm going back and forth
in my head about danger scenarios involving PITR or crashes inbetween
prepare_truncate() and the actual truncation.
ISTM that we should not do visibilitymap_prepare_truncate() during recovery
and always WAL log the prepare_truncate() on the primary. Does that sound
sane?
Greetings,
Andres
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-12 19:27 Tomas Vondra <tomas@vondra.me>
parent: Andres Freund <andres@anarazel.de>
1 sibling, 1 reply; 57+ messages in thread
From: Tomas Vondra @ 2026-03-12 19:27 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; Robert Haas <robertmhaas@gmail.com>; Thomas Munro <thomas.munro@gmail.com>; +Cc: Matthias van de Meent <boekewurm+postgres@gmail.com>; Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On 3/12/26 18:05, Andres Freund wrote:
> Hi,
>
> On 2026-03-06 11:49:20 -0500, Robert Haas wrote:
>> On Thu, Mar 5, 2026 at 7:43 PM Andres Freund <andres@anarazel.de> wrote:
>>> Which perhaps also should have emitted an FPI when clearing a bit? But I'm
>>> unsure that that was required at the time. OTOH, it did seem to generate an
>>> FPI for setting a VM bit, so ...
>>
>> After booting up the part of my brain that worked on this back in
>> 2011, I think I reasoned that clearing a bit was idempotent and didn't
>> depend on the prior page state, so that we would be adequate protected
>> without an FPI. I had to add a WAL record to *set* the VM bit, because
>> that wasn't logged at all, but the actual of clearing the VM bit
>> piggybacked on the heap change that necessitated doing so, and having
>> that record also emit an FPI for the VM page didn't seem to me to add
>> anything.
>>
>> So I think technically it's the addition of checksums that turns this
>> into a real bug, because now torn pages are a checksum-failure hazard.
>> However, that would have been extremely hard to notice given that I
>> seem never to have actually documented that reasoning in a comment
>> anywhere.
>
> And then it got a lot worse with incremental backup support...
>
>
>> I don't think we should mess around with trying to make the behavior
>> here conditional on wal_level, checksums, etc. We should just do it
>> all the time to keep the logic simple.
>
> Agreed.
>
>
> I've been working on a fix for this (still pretty raw though). What a mess.
>
>
> As part of validating that I added error checks that disallow reads in the
> startup process for anything other than the FSM fork. Which triggered, due to
> visibilitymap_prepare_truncate() reading a VM page.
>
> Which in turn made me wonder: Is it actually OK for
> visibilitymap_prepare_truncate() to do separate WAL logging from
> XLOG_SMGR_TRUNCATE?
>
> I suspect not, consider this scenario:
>
> 1) primary does XLOG_FPI for the VM page
> 2) checkpoint
> 3) primary does XLOG_SMGR_TRUNCATE
> 4) standby replays XLOG_FPI
> 5) primary performs a restartpoint
> 6) primary replays XLOG_SMGR_TRUNCATE, as part of that it again does
> visibilitymap_prepare_truncate, which dirties the VM page
> 7) standby crashes while writing out the VM page
> 8) standby does recovery and now finds a torn VM page, triggering a checksum
> failure
>
> Perhaps that's kinda ok, because vm_readbuf() uses ZERO_ON_ERROR, so we'd just
> wipe out that region of the VM?
>
Shouldn't it be possible to test this using injection points?
>
> Separately, is it actually sufficient that visibilitymap_prepare_truncate()
> only WAL logs the changes if XLogHintBitIsNeeded()? I'm going back and forth
> in my head about danger scenarios involving PITR or crashes inbetween
> prepare_truncate() and the actual truncation.
>
>
> ISTM that we should not do visibilitymap_prepare_truncate() during recovery
> and always WAL log the prepare_truncate() on the primary. Does that sound
> sane?
>
I'm considering writing a stress test to try to shake out issues in
this. That strategy was pretty successful for the online checksums
patch, where we had a hunch it might not be quite correct- but it was
hard to come up with a reproducer. The stress test failures were very
useful in that they gave us proof of issues and some examples.
But the crucial part is an ability to verify correctness. With the
checksums it's easy enough - just verify checksums / look for checksum
failures in the server log. But what would that be here?
regards
--
Tomas Vondra
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-12 21:26 Andres Freund <andres@anarazel.de>
parent: Tomas Vondra <tomas@vondra.me>
0 siblings, 1 reply; 57+ messages in thread
From: Andres Freund @ 2026-03-12 21:26 UTC (permalink / raw)
To: Tomas Vondra <tomas@vondra.me>; +Cc: Robert Haas <robertmhaas@gmail.com>; Thomas Munro <thomas.munro@gmail.com>; Matthias van de Meent <boekewurm+postgres@gmail.com>; Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
Hi,
On 2026-03-12 20:27:32 +0100, Tomas Vondra wrote:
> > I've been working on a fix for this (still pretty raw though). What a mess.
> >
> >
> > As part of validating that I added error checks that disallow reads in the
> > startup process for anything other than the FSM fork. Which triggered, due to
> > visibilitymap_prepare_truncate() reading a VM page.
> >
> > Which in turn made me wonder: Is it actually OK for
> > visibilitymap_prepare_truncate() to do separate WAL logging from
> > XLOG_SMGR_TRUNCATE?
> >
> > I suspect not, consider this scenario:
> >
> > 1) primary does XLOG_FPI for the VM page
> > 2) checkpoint
> > 3) primary does XLOG_SMGR_TRUNCATE
> > 4) standby replays XLOG_FPI
> > 5) primary performs a restartpoint
> > 6) primary replays XLOG_SMGR_TRUNCATE, as part of that it again does
> > visibilitymap_prepare_truncate, which dirties the VM page
> > 7) standby crashes while writing out the VM page
> > 8) standby does recovery and now finds a torn VM page, triggering a checksum
> > failure
> >
> > Perhaps that's kinda ok, because vm_readbuf() uses ZERO_ON_ERROR, so we'd just
> > wipe out that region of the VM?
> >
>
> Shouldn't it be possible to test this using injection points?
Not trivially, I think? For one, you're not normally going to get a torn
page, even if you kill the server in the right moment, so you're rarely going
to be able to reproduce checksum errors without further infrastructure. But
also, since the failure behaviour is going to be to clear the VM, you'd not
really see bad consequences, I think?
I think we really need some logic during WAL replay that does more
verification of pages that are read in (i.e. do not have an FPI applied) by
the startup process.
- Direct uses XLogReadBufferExtended() should only be allowed with
ZERO_ON_ERROR, as it has no provisions for applying FPIs.
- Unless FPWs are disabled or ZERO_ON_ERROR is used:
- the read in page must have an LSN newer than the last REDO LSN, otherwise
there should have been an FPI
- the page LSN may not have an LSN from the future (i.e BLK_DONE
should never happen with FPWs)
- all FPIs in the record have been applied
A slightly different, but related, thing that we are missing is infrastructure
to detect pages that have been dirtied without associated WAL logging. I
wonder if we could use a bit in the buffer descriptor to track if a page has
been dirtied by anything other than a hint bit write. Or maybe we could
detect it when unlocking the page.
> > Separately, is it actually sufficient that visibilitymap_prepare_truncate()
> > only WAL logs the changes if XLogHintBitIsNeeded()? I'm going back and forth
> > in my head about danger scenarios involving PITR or crashes inbetween
> > prepare_truncate() and the actual truncation.
> >
> >
> > ISTM that we should not do visibilitymap_prepare_truncate() during recovery
> > and always WAL log the prepare_truncate() on the primary. Does that sound
> > sane?
> >
>
> I'm considering writing a stress test to try to shake out issues in
> this.
We could certainly use more of that.
> That strategy was pretty successful for the online checksums patch, where we
> had a hunch it might not be quite correct- but it was hard to come up with a
> reproducer. The stress test failures were very useful in that they gave us
> proof of issues and some examples.
> But the crucial part is an ability to verify correctness.
Unfortunately I don't think we have a particularly good infrastructure for
detecting problems right now :(
The verification ideas I posted above would, I think, help to detect some
issues, but I don't think they'd catch more complicated things.
We have wal_consistency_checking, but as it runs just after the redo routine,
it doesn't catch problems like not including things in the WAL record or
covering related actions in two WAL records that could be separated by a
checkpoint. We really should have a tool that compares a primary and a
replica after doing recovery using the masking infrastructure from
wal_consistency_checking.
> With the checksums it's easy enough - just verify checksums / look for
> checksum failures in the server log. But what would that be here?
Unfortunately that's not even something you can really can rely on, it's
entirely possible to see checksum errors for the FSM without it being a bug,
as it's not WAL logged.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-12 21:54 Tomas Vondra <tomas@vondra.me>
parent: Andres Freund <andres@anarazel.de>
0 siblings, 1 reply; 57+ messages in thread
From: Tomas Vondra @ 2026-03-12 21:54 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; +Cc: Robert Haas <robertmhaas@gmail.com>; Thomas Munro <thomas.munro@gmail.com>; Matthias van de Meent <boekewurm+postgres@gmail.com>; Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On 3/12/26 22:26, Andres Freund wrote:
> Hi,
>
> On 2026-03-12 20:27:32 +0100, Tomas Vondra wrote:
>>> I've been working on a fix for this (still pretty raw though). What a mess.
>>>
>>>
>>> As part of validating that I added error checks that disallow reads in the
>>> startup process for anything other than the FSM fork. Which triggered, due to
>>> visibilitymap_prepare_truncate() reading a VM page.
>>>
>>> Which in turn made me wonder: Is it actually OK for
>>> visibilitymap_prepare_truncate() to do separate WAL logging from
>>> XLOG_SMGR_TRUNCATE?
>>>
>>> I suspect not, consider this scenario:
>>>
>>> 1) primary does XLOG_FPI for the VM page
>>> 2) checkpoint
>>> 3) primary does XLOG_SMGR_TRUNCATE
>>> 4) standby replays XLOG_FPI
>>> 5) primary performs a restartpoint
>>> 6) primary replays XLOG_SMGR_TRUNCATE, as part of that it again does
>>> visibilitymap_prepare_truncate, which dirties the VM page
>>> 7) standby crashes while writing out the VM page
>>> 8) standby does recovery and now finds a torn VM page, triggering a checksum
>>> failure
>>>
>>> Perhaps that's kinda ok, because vm_readbuf() uses ZERO_ON_ERROR, so we'd just
>>> wipe out that region of the VM?
>>>
>>
>> Shouldn't it be possible to test this using injection points?
>
> Not trivially, I think? For one, you're not normally going to get a torn
> page, even if you kill the server in the right moment, so you're rarely going
> to be able to reproduce checksum errors without further infrastructure. But
> also, since the failure behaviour is going to be to clear the VM, you'd not
> really see bad consequences, I think?
>
>
> I think we really need some logic during WAL replay that does more
> verification of pages that are read in (i.e. do not have an FPI applied) by
> the startup process.
>
> - Direct uses XLogReadBufferExtended() should only be allowed with
> ZERO_ON_ERROR, as it has no provisions for applying FPIs.
>
> - Unless FPWs are disabled or ZERO_ON_ERROR is used:
>
> - the read in page must have an LSN newer than the last REDO LSN, otherwise
> there should have been an FPI
>
> - the page LSN may not have an LSN from the future (i.e BLK_DONE
> should never happen with FPWs)
>
> - all FPIs in the record have been applied
>
>
>
> A slightly different, but related, thing that we are missing is infrastructure
> to detect pages that have been dirtied without associated WAL logging. I
> wonder if we could use a bit in the buffer descriptor to track if a page has
> been dirtied by anything other than a hint bit write. Or maybe we could
> detect it when unlocking the page.
>
>
>>> Separately, is it actually sufficient that visibilitymap_prepare_truncate()
>>> only WAL logs the changes if XLogHintBitIsNeeded()? I'm going back and forth
>>> in my head about danger scenarios involving PITR or crashes inbetween
>>> prepare_truncate() and the actual truncation.
>>>
>>>
>>> ISTM that we should not do visibilitymap_prepare_truncate() during recovery
>>> and always WAL log the prepare_truncate() on the primary. Does that sound
>>> sane?
>>>
>>
>> I'm considering writing a stress test to try to shake out issues in
>> this.
>
> We could certainly use more of that.
>
>
>> That strategy was pretty successful for the online checksums patch, where we
>> had a hunch it might not be quite correct- but it was hard to come up with a
>> reproducer. The stress test failures were very useful in that they gave us
>> proof of issues and some examples.
>
>
>> But the crucial part is an ability to verify correctness.
>
> Unfortunately I don't think we have a particularly good infrastructure for
> detecting problems right now :(
>
>
> The verification ideas I posted above would, I think, help to detect some
> issues, but I don't think they'd catch more complicated things.
>
>
> We have wal_consistency_checking, but as it runs just after the redo routine,
> it doesn't catch problems like not including things in the WAL record or
> covering related actions in two WAL records that could be separated by a
> checkpoint. We really should have a tool that compares a primary and a
> replica after doing recovery using the masking infrastructure from
> wal_consistency_checking.
>
Silly idea - wouldn't it be possible to detect this particular issue
solely based on WAL? We could even have an off-line tool that reads WAL
and checks that we have all FPIs for all the changes.
>
>> With the checksums it's easy enough - just verify checksums / look for
>> checksum failures in the server log. But what would that be here?
>
> Unfortunately that's not even something you can really can rely on, it's
> entirely possible to see checksum errors for the FSM without it being a bug,
> as it's not WAL logged.
>
I did not mean to imply that any checksum failure is necessarily an
outright bug. But I think this kind of issues being "normal" is a hint
maybe not WAL-logging FSM is not such a good design choice anymore.
regards
--
Tomas Vondra
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-12 23:34 Andres Freund <andres@anarazel.de>
parent: Tomas Vondra <tomas@vondra.me>
0 siblings, 0 replies; 57+ messages in thread
From: Andres Freund @ 2026-03-12 23:34 UTC (permalink / raw)
To: Tomas Vondra <tomas@vondra.me>; +Cc: Robert Haas <robertmhaas@gmail.com>; Thomas Munro <thomas.munro@gmail.com>; Matthias van de Meent <boekewurm+postgres@gmail.com>; Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
Hi,
On 2026-03-12 22:54:25 +0100, Tomas Vondra wrote:
> >> But the crucial part is an ability to verify correctness.
> >
> > Unfortunately I don't think we have a particularly good infrastructure for
> > detecting problems right now :(
> >
> >
> > The verification ideas I posted above would, I think, help to detect some
> > issues, but I don't think they'd catch more complicated things.
> >
> >
> > We have wal_consistency_checking, but as it runs just after the redo routine,
> > it doesn't catch problems like not including things in the WAL record or
> > covering related actions in two WAL records that could be separated by a
> > checkpoint. We really should have a tool that compares a primary and a
> > replica after doing recovery using the masking infrastructure from
> > wal_consistency_checking.
> >
>
> Silly idea - wouldn't it be possible to detect this particular issue
> solely based on WAL? We could even have an off-line tool that reads WAL
> and checks that we have all FPIs for all the changes.
Afaict the problem is that you can't easily see that from the WAL in case of
bugs. E.g. for the case at hand, we didn't include block references for the
cleared VM pages in the WAL record. So such a tool would need to know about
the XLH_INSERT_ALL_VISIBLE_CLEARED flag etc.
> >> With the checksums it's easy enough - just verify checksums / look for
> >> checksum failures in the server log. But what would that be here?
> >
> > Unfortunately that's not even something you can really can rely on, it's
> > entirely possible to see checksum errors for the FSM without it being a bug,
> > as it's not WAL logged.
> >
>
> I did not mean to imply that any checksum failure is necessarily an
> outright bug. But I think this kind of issues being "normal" is a hint
> maybe not WAL-logging FSM is not such a good design choice anymore.
I agree we'll eventually have to fix this, I just don't entirely know how.
Perhaps it'd be good enough to essentially treat the FSM as hint bits,
i.e. WAL log the first modification of an FSM page in a checkpoint, if
checksums/log-hints is on.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-03-17 16:06 Robert Haas <robertmhaas@gmail.com>
parent: Andres Freund <andres@anarazel.de>
1 sibling, 0 replies; 57+ messages in thread
From: Robert Haas @ 2026-03-17 16:06 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; +Cc: Thomas Munro <thomas.munro@gmail.com>; Matthias van de Meent <boekewurm+postgres@gmail.com>; Melanie Plageman <melanieplageman@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Thu, Mar 12, 2026 at 1:05 PM Andres Freund <andres@anarazel.de> wrote:
> Which in turn made me wonder: Is it actually OK for
> visibilitymap_prepare_truncate() to do separate WAL logging from
> XLOG_SMGR_TRUNCATE?
I mean, I think this is just a blatant violation of the usual practice
around write-ahead logging. We're hoping that the change we're doing
here will be atomic with a later change in a separate critical
section. I don't see how that's ever going to work out. Functions with
"prepare" in the name should be limited to doing things that can be
done ahead of the operation for which they are preparing. The actual
operation needs to all happen in one critical section after all
preparation is complete.
> Separately, is it actually sufficient that visibilitymap_prepare_truncate()
> only WAL logs the changes if XLogHintBitIsNeeded()? I'm going back and forth
> in my head about danger scenarios involving PITR or crashes inbetween
> prepare_truncate() and the actual truncation.
Since we don't set the page LSN, the changes made in that critical
section can make it to disk even if we don't manage to replay the
truncate record and fail to make it to disk even if we do. That can
only be OK if those changes are never critical. The only thing we're
doing here is clearing VM bits, so if that happens when it shouldn't,
nothing should break, except maybe synchronization between primaries
and standbys. But if it does happen when it shouldn't, that's
potentially data-corrupting: the relation can be re-extended, and now
you've got pages where the VM bit in the page is clear and the VM bit
in the map is set, which is never OK.
> ISTM that we should not do visibilitymap_prepare_truncate() during recovery
> and always WAL log the prepare_truncate() on the primary. Does that sound
> sane?
I think the real need here is, um, to just following the regular
WAL-logging rules. All the things that need to happen as part of a
single operation need to be done in one critical section while holding
one set of buffer locks all at the same time, and the WAL record needs
to be written in that same critical section. Or alternatively several
operations that each follow these rules and together add up to the
same thing. For example, it would be fine to treat clearing the
tailing bits of the last partial page as a separate operation that
happens before the actual truncation and that therefore has its own
WAL record, buffer locks, and critical section, provided that you can
keep any of those bits from being set again before the actual
truncation happens, e.g. by hanging onto the buffer lock. During
replay, there can't be any concurrent writers, so replay can be fully
consecutive.
I feel like what's basically happened here is that an attempt was made
to hide too many of the details under the visibilitymap_* layer.
That's nice for abstraction, but it's not so great for correctness.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-04-30 21:44 Melanie Plageman <melanieplageman@gmail.com>
parent: Andres Freund <andres@anarazel.de>
1 sibling, 3 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-04-30 21:44 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; +Cc: Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
On Thu, Mar 5, 2026 at 4:01 PM Andres Freund <andres@anarazel.de> wrote:
>
> On 2026-03-05 15:38:24 -0500, Andres Freund wrote:
>
> > There's explicit code for ignoring the FSM, but I don't see the same for the
> > VM. And that makes sense: VM changes are mostly WAL logged, just not
> > completely / generically (i.e. this complaint), whereas FSM changes are not
> > WAL logged at all.
>
> Unfortunately I can confirm that incremental backups end up with an outdated
> VM.
>
> An unfortunate kicker: It looks like verify_heapam() doesn't even notice :(.
Attached is a patch set to fix the issue based largely on the work you
started on your branch. I attached the version targeting master/19
which is prefixed with v1_PGMASTER and the version targeting 18,
prefixed v1_PG18. The pg 18 changes aren't a straight cherry-pick to
17 (the earliest I'll backpatch because that was when incremental
backup was introduced) because the redo functions live in a different
file in 18 than in 17, but I want to avoid discussing three different
versions of this patch set on this thread.
The backpatched changes are different for a few reasons, but the
biggest difference from a review standpoint is that in pg18, the redo
routines can read WAL in the old format or the new format, so that
people can reasonably upgrade to the new minor version.
I added a TAP test that does something like Andres' incremental backup
repro but tests more variations. Because init_from_backup() runs
pg_combinebackup with the --debug flag, the log output is very verbose
(i.e. every single copied file is logged). I suggest we turn it off by
default in tests. I included a patch that does that. It isn't required
to commit my test, but my test does produce 5000 lines of regression
log output which seems...not ideal.
The patch set targeting master includes a few more changes aimed at
catching bugs like this in the future. 0005 adds a check to
verify_heapam() that PD_ALL_VISIBLE is never clear when the VM is set.
0006 stops masking PD_ALL_VISIBLE during WAL consistency checking. And
0007 is a version of a patch Andres started which validates that every
block registered with a WAL record is read during recovery.
- Melanie
Attachments: download all (10 files, .tar.gz)
v1_PGMASTER-0001-Make-VM-clear-take-a-RelFileLocator-and-.patch (text/x-patch, 10.0K) download
From 9b8a4316e390505b9b1600debb84eaabe587683d Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Mon, 27 Apr 2026 17:30:17 -0400
Subject: [PATCH v1_PGMASTER 1/7] Make VM clear take a RelFileLocator and not
fake relcache
This brings it in line with visibilitymap_set(). An upcoming commit will
start reading visibility map buffers with XLogReadBuffer*() functions,
so we no longer require a relation object to pin the visibility map,
allowing us to remove this fake relcache business altogether.
While we're at it, remove a spurious const qualifier from the
RelFileLocator parameter to visibilitymap_set().
Author: Melanie Plageman <melanieplageman@gmail.com>
---
contrib/pg_surgery/heap_surgery.c | 2 +-
src/backend/access/heap/heapam.c | 16 ++++++++--------
src/backend/access/heap/heapam_xlog.c | 14 +++++++-------
src/backend/access/heap/pruneheap.c | 2 +-
src/backend/access/heap/visibilitymap.c | 9 ++++++---
src/include/access/visibilitymap.h | 4 ++--
6 files changed, 25 insertions(+), 22 deletions(-)
diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c
index ae4e7c0136c..b82c29e7cbf 100644
--- a/contrib/pg_surgery/heap_surgery.c
+++ b/contrib/pg_surgery/heap_surgery.c
@@ -267,7 +267,7 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
if (PageIsAllVisible(page))
{
PageClearAllVisible(page);
- visibilitymap_clear(rel, blkno, vmbuf,
+ visibilitymap_clear(rel->rd_locator, blkno, vmbuf,
VISIBILITYMAP_VALID_BITS);
did_modify_vm = true;
}
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index abfd8e8970a..777f8bec2e0 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2063,7 +2063,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
{
all_visible_cleared = true;
PageClearAllVisible(page);
- visibilitymap_clear(relation,
+ visibilitymap_clear(relation->rd_locator,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
@@ -2440,7 +2440,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
{
all_visible_cleared = true;
PageClearAllVisible(page);
- visibilitymap_clear(relation,
+ visibilitymap_clear(relation->rd_locator,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
@@ -2998,7 +2998,7 @@ l1:
{
all_visible_cleared = true;
PageClearAllVisible(page);
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
+ visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
@@ -3832,7 +3832,7 @@ l2:
* worthwhile.
*/
if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
+ visibilitymap_clear(relation->rd_locator, block, vmbuffer,
VISIBILITYMAP_ALL_FROZEN))
cleared_all_frozen = true;
@@ -4064,14 +4064,14 @@ l2:
{
all_visible_cleared = true;
PageClearAllVisible(page);
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
+ visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
if (newbuf != buffer && PageIsAllVisible(newpage))
{
all_visible_cleared_new = true;
PageClearAllVisible(newpage);
- visibilitymap_clear(relation, BufferGetBlockNumber(newbuf),
+ visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(newbuf),
vmbuffer_new, VISIBILITYMAP_VALID_BITS);
}
@@ -5158,7 +5158,7 @@ failed:
/* Clear only the all-frozen bit on visibility map if needed */
if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
+ visibilitymap_clear(relation->rd_locator, block, vmbuffer,
VISIBILITYMAP_ALL_FROZEN))
cleared_all_frozen = true;
@@ -5912,7 +5912,7 @@ l4:
&new_xmax, &new_infomask, &new_infomask2);
if (PageIsAllVisible(BufferGetPage(buf)) &&
- visibilitymap_clear(rel, block, vmbuffer,
+ visibilitymap_clear(rel->rd_locator, block, vmbuffer,
VISIBILITYMAP_ALL_FROZEN))
cleared_all_frozen = true;
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index 9ed7024e814..9dbab095795 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -313,7 +313,7 @@ heap_xlog_delete(XLogReaderState *record)
Buffer vmbuffer = InvalidBuffer;
visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear(target_locator, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -400,7 +400,7 @@ heap_xlog_insert(XLogReaderState *record)
Buffer vmbuffer = InvalidBuffer;
visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear(target_locator, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -531,7 +531,7 @@ heap_xlog_multi_insert(XLogReaderState *record)
Relation reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear(rlocator, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
ReleaseBuffer(vmbuffer);
vmbuffer = InvalidBuffer;
FreeFakeRelcacheEntry(reln);
@@ -749,7 +749,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Buffer vmbuffer = InvalidBuffer;
visibilitymap_pin(reln, oldblk, &vmbuffer);
- visibilitymap_clear(reln, oldblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear(rlocator, oldblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -833,7 +833,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Buffer vmbuffer = InvalidBuffer;
visibilitymap_pin(reln, newblk, &vmbuffer);
- visibilitymap_clear(reln, newblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear(rlocator, newblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -1037,7 +1037,7 @@ heap_xlog_lock(XLogReaderState *record)
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
+ visibilitymap_clear(rlocator, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
@@ -1113,7 +1113,7 @@ heap_xlog_lock_updated(XLogReaderState *record)
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
+ visibilitymap_clear(rlocator, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c
index fdddd23035b..184fb70a24f 100644
--- a/src/backend/access/heap/pruneheap.c
+++ b/src/backend/access/heap/pruneheap.c
@@ -927,7 +927,7 @@ heap_page_fix_vm_corruption(PruneState *prstate, OffsetNumber offnum,
if (do_clear_vm)
{
- visibilitymap_clear(prstate->relation, prstate->block,
+ visibilitymap_clear(prstate->relation->rd_locator, prstate->block,
prstate->vmbuffer,
VISIBILITYMAP_VALID_BITS);
prstate->old_vmbits = 0;
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 4fd470702aa..3b699291b23 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -148,7 +148,8 @@ static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks);
* any I/O. Returns true if any bits have been cleared and false otherwise.
*/
bool
-visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
+visibilitymap_clear(RelFileLocator rlocator,
+ BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
{
BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
@@ -162,7 +163,9 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
Assert(flags != VISIBILITYMAP_ALL_VISIBLE);
#ifdef TRACE_VISIBILITYMAP
- elog(DEBUG1, "vm_clear %s %d", RelationGetRelationName(rel), heapBlk);
+ elog(DEBUG1, "vm_clear %s %d",
+ relpathbackend(rlocator, MyProcNumber, MAIN_FORKNUM).str,
+ heapBlk);
#endif
if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
@@ -254,7 +257,7 @@ visibilitymap_pin_ok(BlockNumber heapBlk, Buffer vmbuf)
void
visibilitymap_set(BlockNumber heapBlk,
Buffer vmBuf, uint8 flags,
- const RelFileLocator rlocator)
+ RelFileLocator rlocator)
{
BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
uint32 mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index e4e0cfa989e..b860c4ef3ca 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -26,14 +26,14 @@
#define VM_ALL_FROZEN(r, b, v) \
((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0)
-extern bool visibilitymap_clear(Relation rel, BlockNumber heapBlk,
+extern bool visibilitymap_clear(RelFileLocator rlocator, BlockNumber heapBlk,
Buffer vmbuf, uint8 flags);
extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk,
Buffer *vmbuf);
extern bool visibilitymap_pin_ok(BlockNumber heapBlk, Buffer vmbuf);
extern void visibilitymap_set(BlockNumber heapBlk,
Buffer vmBuf, uint8 flags,
- const RelFileLocator rlocator);
+ RelFileLocator rlocator);
extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf);
extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen);
extern BlockNumber visibilitymap_prepare_truncate(Relation rel,
--
2.43.0
v1_PGMASTER-0002-Fix-WAL-logging-of-VM-clears.patch (text/x-patch, 31.2K) download
From 1cf86f889057dc8637290e5c32c0df85d2437f48 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Mon, 27 Apr 2026 17:59:11 -0400
Subject: [PATCH v1_PGMASTER 2/7] Fix WAL logging of VM clears
Previously, we failed to register the visibility map buffer when
emitting WAL after clearing the VM bits for heap pages (recovery code
read the VM page normally). This meant that we couldn't emit FPIs of VM
pages when clearing VM bits. While this would not result in a checksum
error because the visibility map is read with RBM_ZERO_ON_ERROR, WAL
summarizer only includes pages which were registered, so a restore from
an incremental backup would not include the modified VM pages -- leading
to data corruption.
Fix this by registering the VM buffer in the WAL record when clearing VM
bits. This also means the VM buffer must be locked throughout the
duration of the critical section where we make changes to the VM and
heap page and emit WAL.
Bumps XLOG_PAGE_MAGIC
Author: Andres Freund <andres@anarazel.de>
Author: Melanie Plageman <melanieplageman@gmail.com>
Backpatch through: 17 when incremental backup was introduced
---
contrib/pg_surgery/heap_surgery.c | 7 +-
src/backend/access/heap/heapam.c | 224 ++++++++++++++++++++----
src/backend/access/heap/heapam_xlog.c | 145 +++++++++------
src/backend/access/heap/pruneheap.c | 4 +-
src/backend/access/heap/visibilitymap.c | 36 ++--
src/bin/pg_walsummary/t/002_blocks.pl | 7 +-
src/include/access/heapam_xlog.h | 15 ++
src/include/access/visibilitymap.h | 4 +-
src/include/access/xlog_internal.h | 2 +-
9 files changed, 330 insertions(+), 114 deletions(-)
diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c
index b82c29e7cbf..80d6f073c4a 100644
--- a/contrib/pg_surgery/heap_surgery.c
+++ b/contrib/pg_surgery/heap_surgery.c
@@ -266,9 +266,10 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
*/
if (PageIsAllVisible(page))
{
+ LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
PageClearAllVisible(page);
visibilitymap_clear(rel->rd_locator, blkno, vmbuf,
- VISIBILITYMAP_VALID_BITS);
+ VISIBILITYMAP_VALID_BITS, InvalidXLogRecPtr);
did_modify_vm = true;
}
}
@@ -331,7 +332,9 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
UnlockReleaseBuffer(buf);
- if (vmbuf != InvalidBuffer)
+ if (did_modify_vm)
+ UnlockReleaseBuffer(vmbuf);
+ else if (BufferIsValid(vmbuf))
ReleaseBuffer(vmbuf);
/* Update the current_start_ptr before moving to the next page. */
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 777f8bec2e0..a97d3ae1cc4 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -60,7 +60,8 @@
static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup,
TransactionId xid, CommandId cid, uint32 options);
static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup,
+ Buffer vmbuffer_old, Buffer newbuf,
+ Buffer vmbuffer_new, HeapTuple oldtup,
HeapTuple newtup, HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared,
bool walLogical);
@@ -2062,10 +2063,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
if (PageIsAllVisible(page))
{
all_visible_cleared = true;
+
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
PageClearAllVisible(page);
visibilitymap_clear(relation->rd_locator,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ vmbuffer, VISIBILITYMAP_VALID_BITS, InvalidXLogRecPtr);
}
/*
@@ -2159,13 +2162,23 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (all_visible_cleared)
+ XLogRegisterBuffer(1, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, info);
PageSetLSN(page, recptr);
+
+ if (all_visible_cleared)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -2439,10 +2452,12 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
+
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
PageClearAllVisible(page);
visibilitymap_clear(relation->rd_locator,
BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ vmbuffer, VISIBILITYMAP_VALID_BITS, InvalidXLogRecPtr);
}
else if (all_frozen_set)
{
@@ -2574,7 +2589,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogBeginInsert();
XLogRegisterData(xlrec, tupledata - scratch.data);
XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
- if (all_frozen_set)
+ if (all_frozen_set || all_visible_cleared)
XLogRegisterBuffer(1, vmbuffer, 0);
XLogRegisterBufData(0, tupledata, totaldatalen);
@@ -2585,7 +2600,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
recptr = XLogInsert(RM_HEAP2_ID, info);
PageSetLSN(page, recptr);
- if (all_frozen_set)
+ if (all_frozen_set || all_visible_cleared)
{
Assert(BufferIsDirty(vmbuffer));
PageSetLSN(BufferGetPage(vmbuffer), recptr);
@@ -2594,7 +2609,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- if (all_frozen_set)
+ if (all_frozen_set || all_visible_cleared)
LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
UnlockReleaseBuffer(buffer);
@@ -2997,9 +3012,10 @@ l1:
if (PageIsAllVisible(page))
{
all_visible_cleared = true;
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
PageClearAllVisible(page);
visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ vmbuffer, VISIBILITYMAP_VALID_BITS, InvalidXLogRecPtr);
}
/* store transaction information of xact deleting the tuple */
@@ -3090,13 +3106,23 @@ l1:
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (all_visible_cleared)
+ XLogRegisterBuffer(1, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE);
PageSetLSN(page, recptr);
+
+ if (all_visible_cleared)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
if (vmbuffer != InvalidBuffer)
@@ -3235,6 +3261,8 @@ heap_update(Relation relation, const ItemPointerData *otid, HeapTuple newtup,
bool key_intact;
bool all_visible_cleared = false;
bool all_visible_cleared_new = false;
+ bool modified_vmbuffer = false;
+ bool modified_vmbuffer_new = false;
bool checked_lockers;
bool locker_remains;
bool id_has_external = false;
@@ -3343,8 +3371,11 @@ heap_update(Relation relation, const ItemPointerData *otid, HeapTuple newtup,
UnlockReleaseBuffer(buffer);
Assert(!have_tuple_lock);
- if (vmbuffer != InvalidBuffer)
+ if (BufferIsValid(vmbuffer))
+ {
ReleaseBuffer(vmbuffer);
+ vmbuffer = InvalidBuffer;
+ }
tmfd->ctid = *otid;
tmfd->xmax = InvalidTransactionId;
tmfd->cmax = InvalidCommandId;
@@ -3647,8 +3678,11 @@ l2:
UnlockReleaseBuffer(buffer);
if (have_tuple_lock)
UnlockTupleTuplock(relation, &(oldtup.t_self), *lockmode);
- if (vmbuffer != InvalidBuffer)
+ if (BufferIsValid(vmbuffer))
+ {
ReleaseBuffer(vmbuffer);
+ vmbuffer = InvalidBuffer;
+ }
*update_indexes = TU_None;
bms_free(hot_attrs);
@@ -3831,10 +3865,15 @@ l2:
* overhead would be unchanged, that doesn't seem necessarily
* worthwhile.
*/
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation->rd_locator, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear(relation->rd_locator, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN, InvalidXLogRecPtr))
+ cleared_all_frozen = true;
+ else
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ }
MarkBufferDirty(buffer);
@@ -3853,12 +3892,23 @@ l2:
xlrec.flags =
cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(1, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (cleared_all_frozen)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
/*
@@ -4059,20 +4109,60 @@ l2:
/* record address of new tuple in t_ctid of old one */
oldtup.t_data->t_ctid = heaptup->t_self;
- /* clear PD_ALL_VISIBLE flags, reset all visibilitymap bits */
- if (PageIsAllVisible(page))
+ /*
+ * Clear PD_ALL_VISIBLE flags and reset visibility map bits for any heap
+ * pages that were all-visible. If there are two heap pages, we may need
+ * to clear VM bits for both.
+ */
+ if (PageIsAllVisible(page) &&
+ newbuf != buffer && PageIsAllVisible(newpage) &&
+ vmbuffer_new == vmbuffer)
{
- all_visible_cleared = true;
+ /*
+ * This is the more complicated case: both the new and old heap pages
+ * are all-visible and both their VM bits are on the same page of the
+ * VM, so we register a single VM buffer as HEAP_UPDATE_BLKREF_VM_NEW
+ * in the WAL record. We must be careful to only lock and register one
+ * buffer, even though we modify it twice -- once for each heap
+ * block's VM bits.
+ */
PageClearAllVisible(page);
- visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
- }
- if (newbuf != buffer && PageIsAllVisible(newpage))
- {
- all_visible_cleared_new = true;
PageClearAllVisible(newpage);
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear(relation->rd_locator, block,
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS, InvalidXLogRecPtr);
visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(newbuf),
- vmbuffer_new, VISIBILITYMAP_VALID_BITS);
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS, InvalidXLogRecPtr);
+ modified_vmbuffer_new = true;
+ all_visible_cleared = true;
+ all_visible_cleared_new = true;
+ }
+ else
+ {
+ /*
+ * In all the remaining cases, we will clear at most one heap block's
+ * VM bits per VM page, so we don't have to worry about locking or
+ * registering the same buffer twice.
+ */
+ if (PageIsAllVisible(page))
+ {
+ PageClearAllVisible(page);
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear(relation->rd_locator, block,
+ vmbuffer, VISIBILITYMAP_VALID_BITS, InvalidXLogRecPtr);
+ modified_vmbuffer = true;
+ all_visible_cleared = true;
+ }
+ if (newbuf != buffer && PageIsAllVisible(newpage))
+ {
+ PageClearAllVisible(newpage);
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(newbuf),
+ vmbuffer_new,
+ VISIBILITYMAP_VALID_BITS, InvalidXLogRecPtr);
+ modified_vmbuffer_new = true;
+ all_visible_cleared_new = true;
+ }
}
if (newbuf != buffer)
@@ -4094,8 +4184,12 @@ l2:
log_heap_new_cid(relation, heaptup);
}
- recptr = log_heap_update(relation, buffer,
- newbuf, &oldtup, heaptup,
+ recptr = log_heap_update(relation,
+ buffer,
+ modified_vmbuffer ? vmbuffer : InvalidBuffer,
+ newbuf,
+ modified_vmbuffer_new ? vmbuffer_new : InvalidBuffer,
+ &oldtup, heaptup,
old_key_tuple,
all_visible_cleared,
all_visible_cleared_new,
@@ -4105,10 +4199,20 @@ l2:
PageSetLSN(newpage, recptr);
}
PageSetLSN(page, recptr);
+
+ if (modified_vmbuffer)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
+ if (modified_vmbuffer_new)
+ PageSetLSN(BufferGetPage(vmbuffer_new), recptr);
}
END_CRIT_SECTION();
+ if (modified_vmbuffer)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ if (modified_vmbuffer_new)
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_UNLOCK);
+
if (newbuf != buffer)
LockBuffer(newbuf, BUFFER_LOCK_UNLOCK);
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
@@ -5157,11 +5261,15 @@ failed:
tuple->t_data->t_ctid = *tid;
/* Clear only the all-frozen bit on visibility map if needed */
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation->rd_locator, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
-
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear(relation->rd_locator, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN, InvalidXLogRecPtr))
+ cleared_all_frozen = true;
+ else
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ }
MarkBufferDirty(*buffer);
@@ -5192,15 +5300,25 @@ failed:
xlrec.flags = cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(1, vmbuffer, 0);
+
/* we don't decode row locks atm, so no need to log the origin */
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (cleared_all_frozen)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
result = TM_Ok;
out_locked:
@@ -5911,10 +6029,15 @@ l4:
xid, mode, false,
&new_xmax, &new_infomask, &new_infomask2);
- if (PageIsAllVisible(BufferGetPage(buf)) &&
- visibilitymap_clear(rel->rd_locator, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(BufferGetPage(buf)))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear(rel->rd_locator, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN, InvalidXLogRecPtr))
+ cleared_all_frozen = true;
+ else
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ }
START_CRIT_SECTION();
@@ -5945,13 +6068,23 @@ l4:
XLogRegisterData(&xlrec, SizeOfHeapLockUpdated);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(1, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_LOCK_UPDATED);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (cleared_all_frozen)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
next:
/* if we find the end of update chain, we're done. */
if (mytup.t_data->t_infomask & HEAP_XMAX_INVALID ||
@@ -8772,8 +8905,9 @@ bottomup_sort_and_shrink(TM_IndexDeleteOp *delstate)
* have modified the buffer(s) and marked them dirty.
*/
static XLogRecPtr
-log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup, HeapTuple newtup,
+log_heap_update(Relation reln, Buffer oldbuf, Buffer vmbuffer_old,
+ Buffer newbuf, Buffer vmbuffer_new,
+ HeapTuple oldtup, HeapTuple newtup,
HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared,
bool walLogical)
@@ -8901,9 +9035,9 @@ log_heap_update(Relation reln, Buffer oldbuf,
if (need_tuple_data)
bufflags |= REGBUF_KEEP_DATA;
- XLogRegisterBuffer(0, newbuf, bufflags);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_NEW, newbuf, bufflags);
if (oldbuf != newbuf)
- XLogRegisterBuffer(1, oldbuf, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_OLD, oldbuf, REGBUF_STANDARD);
XLogRegisterData(&xlrec, SizeOfHeapUpdate);
@@ -8980,6 +9114,20 @@ log_heap_update(Relation reln, Buffer oldbuf,
old_key_tuple->t_len - SizeofHeapTupleHeader);
}
+ /*
+ * Register VM buffers. If the old and new heap pages' VM bits are on the
+ * same VM page, the caller passes only vmbuffer_new (mirroring the heap
+ * page convention where block 0 = new is always registered).
+ */
+ Assert((BufferIsInvalid(vmbuffer_old) && BufferIsInvalid(vmbuffer_new)) ||
+ (vmbuffer_old != vmbuffer_new));
+
+ if (BufferIsValid(vmbuffer_new))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_NEW, vmbuffer_new, 0);
+
+ if (BufferIsValid(vmbuffer_old))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_OLD, vmbuffer_old, 0);
+
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index 9dbab095795..604a21d8f27 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -309,13 +309,14 @@ heap_xlog_delete(XLogReaderState *record)
*/
if (xlrec->flags & XLH_DELETE_ALL_VISIBLE_CLEARED)
{
- Relation reln = CreateFakeRelcacheEntry(target_locator);
Buffer vmbuffer = InvalidBuffer;
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(target_locator, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ if (XLogReadBufferForRedo(record, 1, &vmbuffer) == BLK_NEEDS_REDO)
+ visibilitymap_clear(target_locator,
+ blkno, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
}
if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
@@ -396,13 +397,13 @@ heap_xlog_insert(XLogReaderState *record)
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
{
- Relation reln = CreateFakeRelcacheEntry(target_locator);
Buffer vmbuffer = InvalidBuffer;
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(target_locator, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ if (XLogReadBufferForRedo(record, 1, &vmbuffer) == BLK_NEEDS_REDO)
+ visibilitymap_clear(target_locator, blkno, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
}
/*
@@ -523,18 +524,23 @@ heap_xlog_multi_insert(XLogReaderState *record)
(xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
/*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
+ * If required, clear the VM first, to prevent all-visible temporarily
+ * being set for a heap page that's not all visible anymore.
+ *
+ * If it were possible for XLH_INSERT_ALL_VISIBLE_CLEARED and
+ * XLH_INSERT_ALL_FROZEN_SET to be present in the same record, doing the
+ * XLogReadBufferForRedo() before the PageSetAllVisible() below would be a
+ * problem, as it'd violate the rule that a heap page must never be set
+ * all-visible in the VM while its PD_ALL_VISIBLE is clear.
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
{
- Relation reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(rlocator, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
+ if (XLogReadBufferForRedo(record, 1, &vmbuffer) == BLK_NEEDS_REDO)
+ visibilitymap_clear(rlocator, blkno, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
vmbuffer = InvalidBuffer;
- FreeFakeRelcacheEntry(reln);
}
if (isinit)
@@ -633,7 +639,7 @@ heap_xlog_multi_insert(XLogReaderState *record)
buffer = InvalidBuffer;
/*
- * Read and update the visibility map (VM) block.
+ * Read and update the visibility map (VM) block to set it frozen.
*
* We must always redo VM changes, even if the corresponding heap page
* update was skipped due to the LSN interlock. Each VM block covers
@@ -706,6 +712,9 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
nbuffer;
Page opage,
npage;
+ bool new_cleared,
+ old_cleared;
+ bool same_vm_page;
OffsetNumber offnum;
ItemId lp;
HeapTupleData oldtup;
@@ -739,19 +748,62 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
ItemPointerSet(&newtid, newblk, xlrec->new_offnum);
+ new_cleared = (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0;
+ old_cleared = (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) != 0;
+
/*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
+ * If both the old and new heap pages were all-visible and their VM bits
+ * are on the same VM page, that single VM page is registered as
+ * HEAP_UPDATE_BLKREF_VM_NEW. Clear both heap blocks' VM bits from the
+ * single provided VM buffer.
*/
- if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED)
+ same_vm_page = new_cleared && old_cleared &&
+ !XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_OLD) &&
+ XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_NEW);
+
+ if (same_vm_page)
{
- Relation reln = CreateFakeRelcacheEntry(rlocator);
Buffer vmbuffer = InvalidBuffer;
- visibilitymap_pin(reln, oldblk, &vmbuffer);
- visibilitymap_clear(rlocator, oldblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW, &vmbuffer) ==
+ BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear(rlocator, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ visibilitymap_clear(rlocator, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ }
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else
+ {
+ /*
+ * We can be sure that we need to clear at most one heap page's VM
+ * bits in each registered VM buffer.
+ */
+ if (new_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW, &vmbuffer) ==
+ BLK_NEEDS_REDO)
+ visibilitymap_clear(rlocator, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ if (old_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_OLD, &vmbuffer) ==
+ BLK_NEEDS_REDO)
+ visibilitymap_clear(rlocator, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
}
/*
@@ -823,21 +875,6 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
else
newaction = XLogReadBufferForRedo(record, 0, &nbuffer);
- /*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
- */
- if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, newblk, &vmbuffer);
- visibilitymap_clear(rlocator, newblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
-
/* Deal with new tuple */
if (newaction == BLK_NEEDS_REDO)
{
@@ -1031,16 +1068,14 @@ heap_xlog_lock(XLogReaderState *record)
RelFileLocator rlocator;
Buffer vmbuffer = InvalidBuffer;
BlockNumber block;
- Relation reln;
XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
- reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(rlocator, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ if (XLogReadBufferForRedo(record, 1, &vmbuffer) == BLK_NEEDS_REDO)
+ visibilitymap_clear(rlocator, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN,
+ lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
}
if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
@@ -1107,16 +1142,14 @@ heap_xlog_lock_updated(XLogReaderState *record)
RelFileLocator rlocator;
Buffer vmbuffer = InvalidBuffer;
BlockNumber block;
- Relation reln;
XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
- reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(rlocator, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ if (XLogReadBufferForRedo(record, 1, &vmbuffer) == BLK_NEEDS_REDO)
+ visibilitymap_clear(rlocator, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN, lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
}
if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c
index 184fb70a24f..a9feb16e3ec 100644
--- a/src/backend/access/heap/pruneheap.c
+++ b/src/backend/access/heap/pruneheap.c
@@ -927,9 +927,11 @@ heap_page_fix_vm_corruption(PruneState *prstate, OffsetNumber offnum,
if (do_clear_vm)
{
+ LockBuffer(prstate->vmbuffer, BUFFER_LOCK_EXCLUSIVE);
visibilitymap_clear(prstate->relation->rd_locator, prstate->block,
prstate->vmbuffer,
- VISIBILITYMAP_VALID_BITS);
+ VISIBILITYMAP_VALID_BITS, InvalidXLogRecPtr);
+ LockBuffer(prstate->vmbuffer, BUFFER_LOCK_UNLOCK);
prstate->old_vmbits = 0;
}
}
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 3b699291b23..03f12a4b180 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -139,22 +139,26 @@
static Buffer vm_readbuf(Relation rel, BlockNumber blkno, bool extend);
static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks);
-
/*
* visibilitymap_clear - clear specified bits for one page in visibility map
*
- * You must pass a buffer containing the correct map page to this function.
- * Call visibilitymap_pin first to pin the right one. This function doesn't do
- * any I/O. Returns true if any bits have been cleared and false otherwise.
+ * You must pass a buffer containing the correct map page to this function,
+ * which already needs to be pinned and locked exclusively.
+ *
+ * This function doesn't do any I/O. Returns true if any bits have been
+ * cleared and false otherwise.
*/
bool
-visibilitymap_clear(RelFileLocator rlocator,
- BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
+visibilitymap_clear(RelFileLocator rlocator, BlockNumber heapBlk,
+ Buffer vmbuf, uint8 flags, XLogRecPtr lsn)
{
- BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
int mapOffset = HEAPBLK_TO_OFFSET(heapBlk);
+#ifdef USE_ASSERT_CHECKING
+ BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
+#endif
uint8 mask = flags << mapOffset;
+ Page page;
char *map;
bool cleared = false;
@@ -168,11 +172,11 @@ visibilitymap_clear(RelFileLocator rlocator,
heapBlk);
#endif
- if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
- elog(ERROR, "wrong buffer passed to visibilitymap_clear");
+ Assert(BufferIsValid(vmbuf) && BufferGetBlockNumber(vmbuf) == mapBlock);
+ Assert(BufferIsLockedByMeInMode(vmbuf, BUFFER_LOCK_EXCLUSIVE));
- LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
- map = PageGetContents(BufferGetPage(vmbuf));
+ page = BufferGetPage(vmbuf);
+ map = PageGetContents(page);
if (map[mapByte] & mask)
{
@@ -182,7 +186,15 @@ visibilitymap_clear(RelFileLocator rlocator,
cleared = true;
}
- LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
+ if (XLogRecPtrIsValid(lsn))
+ {
+ /*
+ * Outside of recovery we won't have an LSN to stamp, as we make page
+ * changes before emitting WAL
+ */
+ Assert(InRecovery);
+ PageSetLSN(page, lsn);
+ }
return cleared;
}
diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl
index f5fe94f9f15..3fcf6c1041c 100644
--- a/src/bin/pg_walsummary/t/002_blocks.pl
+++ b/src/bin/pg_walsummary/t/002_blocks.pl
@@ -93,13 +93,14 @@ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
split(m@/@, $end_lsn);
ok(-f $filename, "WAL summary file exists");
-# Run pg_walsummary on it. We expect exactly two blocks to be modified,
-# block 0 and one other.
+# Run pg_walsummary on it. We expect exactly three blocks to be modified,
+# block 0 (old tuple), another block (new tuple) and the block for the VM.
my ($stdout, $stderr) = run_command([ 'pg_walsummary', '-i', $filename ]);
note($stdout);
@lines = split(/\n/, $stdout);
like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified");
+like($stdout, qr/FORK vm: block 0$/m, "stdout shows VM block 0 modified");
is($stderr, '', 'stderr is empty');
-is(0 + @lines, 2, "UPDATE modified 2 blocks");
+is(0 + @lines, 3, "UPDATE modified 3 blocks");
done_testing();
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index fdca7d821c8..4ce028f756d 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -216,7 +216,22 @@ typedef struct xl_multi_insert_tuple
* included even if a full-page image was taken.
*
* Backup blk 1: old page, if different. (no data, just a reference to the blk)
+ *
+ * Backup blk 2: VM page covering the new heap page. Registered whenever
+ * XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED is set. Also covers the old
+ * heap page's VM bits when both heap pages map to the same VM
+ * page.
+ *
+ * Backup blk 3: VM page covering the old heap page. Only registered when
+ * XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED is set and the old heap
+ * page's VM bits are on a different VM page from the new heap
+ * page's.
*/
+#define HEAP_UPDATE_BLKREF_NEW 0
+#define HEAP_UPDATE_BLKREF_OLD 1
+#define HEAP_UPDATE_BLKREF_VM_NEW 2
+#define HEAP_UPDATE_BLKREF_VM_OLD 3
+
typedef struct xl_heap_update
{
TransactionId old_xmax; /* xmax of the old tuple */
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index b860c4ef3ca..f183c11a296 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -15,6 +15,7 @@
#define VISIBILITYMAP_H
#include "access/visibilitymapdefs.h"
+#include "access/xlogdefs.h"
#include "storage/block.h"
#include "storage/buf.h"
#include "storage/relfilelocator.h"
@@ -27,7 +28,8 @@
((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0)
extern bool visibilitymap_clear(RelFileLocator rlocator, BlockNumber heapBlk,
- Buffer vmbuf, uint8 flags);
+ Buffer vmbuf, uint8 flags, XLogRecPtr lsn);
+
extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk,
Buffer *vmbuf);
extern bool visibilitymap_pin_ok(BlockNumber heapBlk, Buffer vmbuf);
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 13ae3ad4fbb..55663e6f4af 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -32,7 +32,7 @@
/*
* Each page of XLOG file has a header like this:
*/
-#define XLOG_PAGE_MAGIC 0xD11F /* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD120 /* can be used as WAL version indicator */
typedef struct XLogPageHeaderData
{
--
2.43.0
v1_PGMASTER-0003-Turn-off-debug-for-pg_combinebackup-TAP-.patch (text/x-patch, 1.9K) download
From 3c306b20a9bee7ea750f4d9cacae9532ebd67427 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 30 Apr 2026 16:06:06 -0400
Subject: [PATCH v1_PGMASTER 3/7] Turn off debug for pg_combinebackup TAP tests
Having --debug as the default option to pg_combinebackup in
init_from_backup() makes the logs extremely verbose. It seems better to
let developers set an environment variable or pass it as a keyword
argument to init_from_backup() to turn on debugging as they wish.
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index bdc051e6ead..ebc776f7590 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -925,7 +925,11 @@ pathnames that should be used for the target cluster.
To restore from an incremental backup, pass the parameter combine_with_prior
as a reference to an array of prior backup names with which this backup
-is to be combined using pg_combinebackup.
+is to be combined using pg_combinebackup. By default, pg_combinebackup is
+run without --debug. To enable --debug output when diagnosing test
+failures, either pass combine_debug => 1 as a keyword parameter, or set
+the environment variable PG_TEST_PG_COMBINEBACKUP_DEBUG to a true value
+to enable it globally for all callers.
Streaming replication can be enabled on this node by passing the keyword
parameter has_streaming => 1. This is disabled by default.
@@ -976,7 +980,9 @@ sub init_from_backup
}
local %ENV = $self->_get_env();
- my @combineargs = ('pg_combinebackup', '--debug');
+ my @combineargs = ('pg_combinebackup');
+ push @combineargs, '--debug'
+ if $params{combine_debug} // $ENV{PG_TEST_PG_COMBINEBACKUP_DEBUG};
if (exists $params{tablespace_map})
{
while (my ($olddir, $newdir) = each %{ $params{tablespace_map} })
--
2.43.0
v1_PGMASTER-0004-Test-that-VM-clear-registers-VM-buffers.patch (text/x-patch, 10.3K) download
From efd17f34929d0f00477788fffe88b6fe5be27653 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 23 Apr 2026 11:13:23 -0400
Subject: [PATCH v1_PGMASTER 4/7] Test that VM clear registers VM buffers
WAL summarizer only includes registered buffers, so you could end up
with data corruption in incremental backups if VM clears were meant to
be covered by the incremental backup.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reported-by: Andres Freund <andres@anarazel.de>
Original repro by Andres
Backpatch through: 17 when incremental backup was introduced
---
src/bin/pg_combinebackup/meson.build | 1 +
.../pg_combinebackup/t/012_vm_consistency.pl | 270 ++++++++++++++++++
2 files changed, 271 insertions(+)
create mode 100644 src/bin/pg_combinebackup/t/012_vm_consistency.pl
diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build
index a35b86f3f59..ba1c8cfa3d0 100644
--- a/src/bin/pg_combinebackup/meson.build
+++ b/src/bin/pg_combinebackup/meson.build
@@ -39,6 +39,7 @@ tests += {
't/009_no_full_file.pl',
't/010_hardlink.pl',
't/011_ib_truncation.pl',
+ 't/012_vm_consistency.pl',
],
}
}
diff --git a/src/bin/pg_combinebackup/t/012_vm_consistency.pl b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
new file mode 100644
index 00000000000..cb03f40bfea
--- /dev/null
+++ b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
@@ -0,0 +1,270 @@
+# Copyright (c) 2021-2026, PostgreSQL Global Development Group
+#
+# Test that heap operations clearing visibility map bits (INSERT, UPDATE,
+# DELETE, SELECT FOR UPDATE, COPY) are correctly registering visibility map
+# buffers, since incremental backups rely on WAL summarizer which only tracks
+# registered buffers.
+#
+# Each operation is tested in a separate vacuum-modify-backup-validate
+# cycle to isolate failures.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir_short();
+my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
+
+# Set up primary with WAL summarization enabled.
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(has_archiving => 1, allows_streaming => 1);
+$primary->append_conf('postgresql.conf', <<EOF);
+summarize_wal = on
+autovacuum = off
+EOF
+$primary->start;
+
+$primary->safe_psql('postgres', q{CREATE EXTENSION pg_visibility});
+
+# Create tables for each test case.
+$primary->safe_psql('postgres', <<SQL);
+CREATE TABLE vm_insert_test (id int);
+INSERT INTO vm_insert_test DEFAULT VALUES;
+
+CREATE TABLE vm_delete_test (id int);
+INSERT INTO vm_delete_test VALUES (1), (2);
+
+CREATE TABLE vm_update_test (id INT, val TEXT);
+INSERT INTO vm_update_test SELECT 1, 'same page';
+INSERT INTO vm_update_test SELECT i, repeat('a', 200) FROM generate_series(2, 70) i;
+
+CREATE TABLE vm_lock_test (id int);
+INSERT INTO vm_lock_test VALUES (1), (2);
+
+CREATE TABLE vm_copy_test (id int);
+INSERT INTO vm_copy_test DEFAULT VALUES;
+SQL
+
+my $backup_num = 0;
+my $restore_num = 0;
+
+# Helper: get both all_visible and all_frozen as a two-element list.
+sub get_vm_summary
+{
+ my ($node, $table) = @_;
+ my $result = $node->safe_psql('postgres',
+ "SELECT all_visible, all_frozen FROM pg_visibility_map_summary('$table')");
+ my @vals = split(/\|/, $result);
+ return @vals;
+}
+
+# Helper: take a full backup. Returns ($path, $name).
+sub take_full_backup
+{
+ $backup_num++;
+ my $name = "full_$backup_num";
+ my $path = $primary->backup_dir . "/$name";
+ $primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $path,
+ '--checkpoint' => 'fast',
+ ],
+ "full backup $backup_num");
+ return ($path, $name);
+}
+
+# Helper: take an incremental backup. Returns ($path, $name).
+sub take_incr_backup
+{
+ my ($prior_path) = @_;
+ $backup_num++;
+ my $name = "incr_$backup_num";
+ my $path = $primary->backup_dir . "/$name";
+ $primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $path,
+ '--checkpoint' => 'fast',
+ '--incremental' => $prior_path . '/backup_manifest',
+ ],
+ "incremental backup $backup_num");
+ return ($path, $name);
+}
+
+# Helper: combine two backups, restore, validate VM for the given table,
+# then stop the restored server. Checks both all_visible and all_frozen.
+sub validate_restored_vm
+{
+ my ($full_path, $full_name, $incr_path, $incr_name, $table, $label) = @_;
+
+ $restore_num++;
+
+ my $restored = PostgreSQL::Test::Cluster->new("restored_$restore_num");
+ $restored->init_from_backup($primary, $incr_name,
+ combine_with_prior => [$full_name],
+ combine_mode => $mode);
+ $restored->append_conf('postgresql.conf', <<EOF);
+autovacuum = off
+EOF
+ $restored->start;
+ $restored->safe_psql('postgres',
+ q{CREATE EXTENSION IF NOT EXISTS pg_visibility});
+
+ my ($primary_visible, $primary_frozen) = get_vm_summary($primary, $table);
+ my ($restored_visible, $restored_frozen) = get_vm_summary($restored, $table);
+
+ is($restored_visible, $primary_visible,
+ "$label: restored all_visible count matches primary");
+ is($restored_frozen, $primary_frozen,
+ "$label: restored all_frozen count matches primary");
+
+ my $corrupt_tids = $restored->safe_psql('postgres',
+ "SELECT count(*) FROM pg_check_visible('$table')");
+ is($corrupt_tids, '0',
+ "$label: no VM corruption detected by pg_check_visible");
+
+ $restored->stop;
+}
+
+##
+## Test 1: INSERT clears all-visible and all-frozen bits
+##
+
+$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_insert_test});
+my ($pre_ins_vis, $pre_ins_frz) = get_vm_summary($primary, 'vm_insert_test');
+cmp_ok($pre_ins_vis, '>', 0,
+ "INSERT test: pages are all-visible after vacuum");
+cmp_ok($pre_ins_frz, '>', 0,
+ "INSERT test: pages are all-frozen after vacuum");
+
+my ($full1, $full1_name) = take_full_backup();
+
+$primary->safe_psql('postgres',
+ q{INSERT INTO vm_insert_test VALUES (1)});
+
+my ($post_ins_vis, $post_ins_frz) = get_vm_summary($primary, 'vm_insert_test');
+cmp_ok($post_ins_vis, '<', $pre_ins_vis,
+ "INSERT test: all-visible cleared after insert");
+cmp_ok($post_ins_frz, '<', $pre_ins_frz,
+ "INSERT test: all-frozen cleared after insert");
+
+my ($incr1, $incr1_name) = take_incr_backup($full1);
+validate_restored_vm($full1, $full1_name, $incr1, $incr1_name,
+ 'vm_insert_test', 'INSERT test');
+
+##
+## Test 2: DELETE clears all-visible and all-frozen bits
+##
+
+$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_delete_test});
+my ($pre_del_vis, $pre_del_frz) = get_vm_summary($primary, 'vm_delete_test');
+cmp_ok($pre_del_vis, '>', 0,
+ "DELETE test: pages are all-visible after vacuum");
+cmp_ok($pre_del_frz, '>', 0,
+ "DELETE test: pages are all-frozen after vacuum");
+
+my ($full2, $full2_name) = take_full_backup();
+
+$primary->safe_psql('postgres',
+ q{DELETE FROM vm_delete_test WHERE id = 1});
+
+my ($post_del_vis, $post_del_frz) = get_vm_summary($primary, 'vm_delete_test');
+cmp_ok($post_del_vis, '<', $pre_del_vis,
+ "DELETE test: all-visible cleared after delete");
+cmp_ok($post_del_frz, '<', $pre_del_frz,
+ "DELETE test: all-frozen cleared after delete");
+
+my ($incr2, $incr2_name) = take_incr_backup($full2);
+validate_restored_vm($full2, $full2_name, $incr2, $incr2_name,
+ 'vm_delete_test', 'DELETE test');
+
+##
+## Test 3: UPDATE clears all-visible and all-frozen bits
+##
+
+$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_update_test});
+my ($pre_upd_vis, $pre_upd_frz) = get_vm_summary($primary, 'vm_update_test');
+cmp_ok($pre_upd_vis, '>', 0,
+ "UPDATE test: pages are all-visible after vacuum");
+cmp_ok($pre_upd_frz, '>', 0,
+ "UPDATE test: pages are all-frozen after vacuum");
+
+my ($full3, $full3_name) = take_full_backup();
+
+# This should update the same page
+$primary->safe_psql('postgres',
+ q{UPDATE vm_update_test SET id = 0 WHERE id = 1});
+# At least some of these updates should be cross-page
+$primary->safe_psql('postgres',
+ q{UPDATE vm_update_test SET id = 99 WHERE id > 50});
+
+my ($post_upd_vis, $post_upd_frz) = get_vm_summary($primary, 'vm_update_test');
+cmp_ok($post_upd_vis, '<', $pre_upd_vis,
+ "UPDATE test: all-visible cleared after update");
+cmp_ok($post_upd_frz, '<', $pre_upd_frz,
+ "UPDATE test: all-frozen cleared after update");
+
+my ($incr3, $incr3_name) = take_incr_backup($full3);
+validate_restored_vm($full3, $full3_name, $incr3, $incr3_name,
+ 'vm_update_test', 'UPDATE test');
+
+##
+## Test 4: SELECT FOR UPDATE clears all-frozen bit (but not all-visible)
+##
+## heap_lock_tuple only clears VISIBILITYMAP_ALL_FROZEN, not ALL_VISIBLE
+##
+
+$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_lock_test});
+my ($pre_lock_vis, $pre_lock_frz) = get_vm_summary($primary, 'vm_lock_test');
+cmp_ok($pre_lock_vis, '>', 0,
+ "LOCK test: pages are all-visible after vacuum");
+cmp_ok($pre_lock_frz, '>', 0,
+ "LOCK test: pages are all-frozen after vacuum");
+
+my ($full4, $full4_name) = take_full_backup();
+
+$primary->safe_psql('postgres',
+ q{SELECT * FROM vm_lock_test WHERE id = 1 FOR UPDATE});
+
+my ($post_lock_vis, $post_lock_frz) = get_vm_summary($primary, 'vm_lock_test');
+is($post_lock_vis, $pre_lock_vis,
+ "LOCK test: all-visible unchanged after SELECT FOR UPDATE");
+cmp_ok($post_lock_frz, '<', $pre_lock_frz,
+ "LOCK test: all-frozen cleared after SELECT FOR UPDATE");
+
+my ($incr4, $incr4_name) = take_incr_backup($full4);
+validate_restored_vm($full4, $full4_name, $incr4, $incr4_name,
+ 'vm_lock_test', 'LOCK test');
+
+##
+## Test 5: COPY clears all-visible and all-frozen bits
+##
+
+$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_copy_test});
+my ($pre_copy_vis, $pre_copy_frz) = get_vm_summary($primary, 'vm_copy_test');
+cmp_ok($pre_copy_vis, '>', 0,
+ "COPY test: pages are all-visible after vacuum");
+cmp_ok($pre_copy_frz, '>', 0,
+ "COPY test: pages are all-frozen after vacuum");
+
+my ($full5, $full5_name) = take_full_backup();
+
+$primary->safe_psql('postgres',
+ "COPY vm_copy_test FROM PROGRAM 'echo 42'");
+
+my ($post_copy_vis, $post_copy_frz) = get_vm_summary($primary, 'vm_copy_test');
+cmp_ok($post_copy_vis, '<', $pre_copy_vis,
+ "COPY test: all-visible cleared after COPY");
+cmp_ok($post_copy_frz, '<', $pre_copy_frz,
+ "COPY test: all-frozen cleared after COPY");
+
+my ($incr5, $incr5_name) = take_incr_backup($full5);
+validate_restored_vm($full5, $full5_name, $incr5, $incr5_name,
+ 'vm_copy_test', 'COPY test');
+
+$primary->stop;
+
+done_testing();
--
2.43.0
v1_PGMASTER-0005-Add-VM-corruption-check-to-verify_heapam.patch (text/x-patch, 1.9K) download
From bb4c03a887f8ec5395ce37c09279a0786712b06c Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 10:20:20 -0400
Subject: [PATCH v1_PGMASTER 5/7] Add VM corruption check to verify_heapam()
pg_check_visible() can catch cases were tuples are not visible to all
and the VM is set, but it doesn't specifically check page hints. This
doesn't check, for example, an INSERT failing to clear PD_ALL_VISIBLE.
This also helps users who have access to amcheck but not pg_visibility
to diagnose corruption.
Author: Melanie Plageman <melanieplageman@gmail.com>
---
contrib/amcheck/verify_heapam.c | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 20ff58aa782..0e023871566 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -481,6 +481,7 @@ verify_heapam(PG_FUNCTION_ARGS)
while ((ctx.buffer = read_stream_next_buffer(stream, NULL)) != InvalidBuffer)
{
+ uint8 vmbits;
OffsetNumber maxoff;
OffsetNumber predecessor[MaxOffsetNumber];
OffsetNumber successor[MaxOffsetNumber];
@@ -499,6 +500,21 @@ verify_heapam(PG_FUNCTION_ARGS)
ctx.blkno = BufferGetBlockNumber(ctx.buffer);
ctx.page = BufferGetPage(ctx.buffer);
+ /*
+ * It is corruption of the page-level PD_ALL_VISIBLE flag is clear and
+ * the visibility map bits corresponding to this heap page are set.
+ */
+ vmbits = visibilitymap_get_status(ctx.rel, ctx.blkno, &vmbuffer);
+
+ if (!PageIsAllVisible(ctx.page) &&
+ (vmbits & VISIBILITYMAP_ALL_VISIBLE) != 0)
+ {
+ ctx.offnum = InvalidOffsetNumber;
+ ctx.attnum = -1;
+ report_corruption(&ctx,
+ psprintf("page is not marked all-visible in page header but visibility map bit is set"));
+ }
+
/* Perform tuple checks */
maxoff = PageGetMaxOffsetNumber(ctx.page);
for (ctx.offnum = FirstOffsetNumber; ctx.offnum <= maxoff;
--
2.43.0
v1_PGMASTER-0006-Stop-masking-PD_ALL_VISIBLE-when-checkin.patch (text/x-patch, 1.4K) download
From 2be0dee6aa408752200ef9adfa9bf6a94716b4b2 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 10:33:46 -0400
Subject: [PATCH v1_PGMASTER 6/7] Stop masking PD_ALL_VISIBLE when checking WAL
consistency
Setting and clearing PD_ALL_VISIBLE is WAL-logged, so there is no need
to mask it for the purposes of WAL consistency checking.
There is one minor exception, which is that when fixing VM corruption,
we clear PD_ALL_VISIBLE on the heap page without specifically emitting
WAL. This is a situation where corruption has already happened, however,
so it is okay if WAL consistency checking flags this as well.
Stop masking PD_ALL_VISIBLE, as doing so can only conceal real bugs.
Author: Melanie Plageman <melanieplageman@gmail.com>
---
src/backend/access/common/bufmask.c | 6 ------
1 file changed, 6 deletions(-)
diff --git a/src/backend/access/common/bufmask.c b/src/backend/access/common/bufmask.c
index 5f63d04c9cb..93e34ea38b8 100644
--- a/src/backend/access/common/bufmask.c
+++ b/src/backend/access/common/bufmask.c
@@ -53,12 +53,6 @@ mask_page_hint_bits(Page page)
/* Ignore PD_PAGE_FULL and PD_HAS_FREE_LINES flags, they are just hints. */
PageClearFull(page);
PageClearHasFreeLinePointers(page);
-
- /*
- * PD_ALL_VISIBLE is masked during WAL consistency checking. XXX: It is
- * worth investigating if we could stop doing this.
- */
- PageClearAllVisible(page);
}
/*
--
2.43.0
v1_PGMASTER-0007-xlog-Verify-all-block-references-are-use.patch (text/x-patch, 3.8K) download
From a8e41a30b323447b023f0cf5a3f26a3f51a9c80f Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 12:08:26 -0400
Subject: [PATCH v1_PGMASTER 7/7] xlog: Verify all block references are used
during replay
It's unclear if this makes sense to actually commit. It requires that
blocks read during recovery go through XLogReadBufferForRedoExtended()
and has a special case for handling XLOG_FPI_FOR_HINT when full page
writes are disabled.
Author: Andres Freund <andres@anarazel.de>
Co-authored-by: Melanie Plageman <melanieplageman@gmail.com>
---
src/backend/access/transam/xlog.c | 10 ++++++++++
src/backend/access/transam/xlogreader.c | 4 ++++
src/backend/access/transam/xlogrecovery.c | 9 +++++++++
src/backend/access/transam/xlogutils.c | 6 ++++++
src/include/access/xlogreader.h | 4 ++++
5 files changed, 33 insertions(+)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 18d5dee06e0..0731439df90 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9085,6 +9085,16 @@ xlog_redo(XLogReaderState *record)
{
if (info == XLOG_FPI)
elog(ERROR, "XLOG_FPI record did not contain a full-page image");
+
+#ifdef USE_ASSERT_CHECKING
+
+ /*
+ * If full_page_writes are disabled, we don't want to error
+ * out because we didn't read the block.
+ */
+ if (info == XLOG_FPI_FOR_HINT)
+ record->record->blocks[block_id].used_read = true;
+#endif
continue;
}
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 8849610db00..63747aec566 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1794,6 +1794,10 @@ DecodeXLogRecord(XLogReaderState *state,
blk = &decoded->blocks[block_id];
blk->in_use = true;
+#ifdef USE_ASSERT_CHECKING
+ blk->used_read = false;
+#endif
+
blk->apply_image = false;
COPY_HEADER_FIELD(&fork_flags, sizeof(uint8));
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index c236e2b7969..4e1d50f642d 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -1962,6 +1962,15 @@ ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *repl
/* Now apply the WAL record itself */
GetRmgr(record->xl_rmid).rm_redo(xlogreader);
+#ifdef USE_ASSERT_CHECKING
+ for (int block_id = 0; block_id <= xlogreader->record->max_block_id; block_id++)
+ {
+ DecodedBkpBlock *blk = &xlogreader->record->blocks[block_id];
+
+ Assert(!blk->in_use || blk->used_read);
+ }
+#endif
+
/*
* After redo, check whether the backup pages associated with the WAL
* record are consistent with the existing pages. This check is done only
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 5fbe39133b8..979d6465159 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -359,6 +359,12 @@ XLogReadBufferForRedoExtended(XLogReaderState *record,
block_id);
}
+#ifdef USE_ASSERT_CHECKING
+ /* Shouldn't have multiple read references to block */
+ Assert(!record->record->blocks[block_id].used_read);
+ record->record->blocks[block_id].used_read = true;
+#endif
+
/*
* Make sure that if the block is marked with WILL_INIT, the caller is
* going to initialize it. And vice versa.
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 97eae2c1dab..fd2119005e8 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -145,6 +145,10 @@ typedef struct
bool has_data;
char *data;
uint16 data_len;
+
+#ifdef USE_ASSERT_CHECKING
+ bool used_read;
+#endif
} DecodedBkpBlock;
/*
--
2.43.0
v1_PG18-0001-Fix-WAL-logging-of-VM-clears.patch (text/x-patch, 31.0K) download
From 8ef81094e39a34d845b1a40cad7de469c8501bee Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 12:53:49 -0400
Subject: [PATCH v1_PG18 1/3] Fix WAL logging of VM clears
Previously, we failed to register the visibility map buffer when
emitting WAL after clearing the VM bits for heap pages (recovery code
read the VM page normally). This meant that we couldn't emit FPIs of VM
pages when clearing VM bits. While this would not result in a checksum
error because the visibility map is read with RBM_ZERO_ON_ERROR, WAL
summarizer only includes pages which were registered, so a restore from
an incremental backup would not include the modified VM pages -- leading
to data corruption.
Fix this by registering the VM buffer in the WAL record when clearing VM
bits. This also means the VM buffer must be locked throughout the
duration of the critical section where we make changes to the VM and
heap page and emit WAL.
---
contrib/pg_surgery/heap_surgery.c | 10 +-
src/backend/access/heap/heapam.c | 233 +++++++++++++++++++----
src/backend/access/heap/heapam_xlog.c | 235 ++++++++++++++++++++----
src/backend/access/heap/visibilitymap.c | 22 ++-
src/bin/pg_walsummary/t/002_blocks.pl | 7 +-
src/include/access/heapam_xlog.h | 15 ++
src/include/access/visibilitymap.h | 2 +
7 files changed, 438 insertions(+), 86 deletions(-)
diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c
index 3e86283beb7..d7e4e051fff 100644
--- a/contrib/pg_surgery/heap_surgery.c
+++ b/contrib/pg_surgery/heap_surgery.c
@@ -266,9 +266,10 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
*/
if (PageIsAllVisible(page))
{
+ LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
PageClearAllVisible(page);
- visibilitymap_clear(rel, blkno, vmbuf,
- VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear_locked(rel, blkno, vmbuf,
+ VISIBILITYMAP_VALID_BITS);
did_modify_vm = true;
}
}
@@ -331,7 +332,10 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
UnlockReleaseBuffer(buf);
- if (vmbuf != InvalidBuffer)
+
+ if (did_modify_vm)
+ UnlockReleaseBuffer(vmbuf);
+ else if (BufferIsValid(vmbuf))
ReleaseBuffer(vmbuf);
/* Update the current_start_ptr before moving to the next page. */
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 6203e3d7f8d..38137f5dc8e 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -58,7 +58,8 @@
static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup,
TransactionId xid, CommandId cid, int options);
static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup,
+ Buffer vmbuffer_old, Buffer newbuf,
+ Buffer vmbuffer_new, HeapTuple oldtup,
HeapTuple newtup, HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared);
#ifdef USE_ASSERT_CHECKING
@@ -2135,10 +2136,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
if (PageIsAllVisible(BufferGetPage(buffer)))
{
all_visible_cleared = true;
+
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
PageClearAllVisible(BufferGetPage(buffer));
- visibilitymap_clear(relation,
- ItemPointerGetBlockNumber(&(heaptup->t_self)),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear_locked(relation,
+ ItemPointerGetBlockNumber(&(heaptup->t_self)),
+ vmbuffer, VISIBILITYMAP_VALID_BITS);
}
/*
@@ -2228,13 +2231,22 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (all_visible_cleared)
+ XLogRegisterBuffer(1, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, info);
PageSetLSN(page, recptr);
+
+ if (all_visible_cleared)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -2503,10 +2515,11 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
PageClearAllVisible(page);
- visibilitymap_clear(relation,
- BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear_locked(relation,
+ BufferGetBlockNumber(buffer),
+ vmbuffer, VISIBILITYMAP_VALID_BITS);
}
else if (all_frozen_set)
PageSetAllVisible(page);
@@ -2620,6 +2633,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogBeginInsert();
XLogRegisterData(xlrec, tupledata - scratch.data);
XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
+ if (all_visible_cleared)
+ XLogRegisterBuffer(1, vmbuffer, 0);
XLogRegisterBufData(0, tupledata, totaldatalen);
@@ -2629,10 +2644,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
recptr = XLogInsert(RM_HEAP2_ID, info);
PageSetLSN(page, recptr);
+ if (all_visible_cleared)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
/*
* If we've frozen everything on the page, update the visibilitymap.
* We're already holding pin on the vmbuffer.
@@ -3050,9 +3070,10 @@ l1:
if (PageIsAllVisible(page))
{
all_visible_cleared = true;
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
PageClearAllVisible(page);
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear_locked(relation, BufferGetBlockNumber(buffer),
+ vmbuffer, VISIBILITYMAP_VALID_BITS);
}
/* store transaction information of xact deleting the tuple */
@@ -3133,13 +3154,23 @@ l1:
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (all_visible_cleared)
+ XLogRegisterBuffer(1, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE);
PageSetLSN(page, recptr);
+
+ if (all_visible_cleared)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
if (vmbuffer != InvalidBuffer)
@@ -3257,7 +3288,8 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
HeapTuple heaptup;
HeapTuple old_key_tuple = NULL;
bool old_key_copied = false;
- Page page;
+ Page page,
+ newpage;
BlockNumber block;
MultiXactStatus mxact_status;
Buffer buffer,
@@ -3274,6 +3306,8 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
bool key_intact;
bool all_visible_cleared = false;
bool all_visible_cleared_new = false;
+ bool modified_vmbuffer = false;
+ bool modified_vmbuffer_new = false;
bool checked_lockers;
bool locker_remains;
bool id_has_external = false;
@@ -3382,8 +3416,11 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
UnlockReleaseBuffer(buffer);
Assert(!have_tuple_lock);
- if (vmbuffer != InvalidBuffer)
+ if (BufferIsValid(vmbuffer))
+ {
ReleaseBuffer(vmbuffer);
+ vmbuffer = InvalidBuffer;
+ }
tmfd->ctid = *otid;
tmfd->xmax = InvalidTransactionId;
tmfd->cmax = InvalidCommandId;
@@ -3686,8 +3723,11 @@ l2:
UnlockReleaseBuffer(buffer);
if (have_tuple_lock)
UnlockTupleTuplock(relation, &(oldtup.t_self), *lockmode);
- if (vmbuffer != InvalidBuffer)
+ if (BufferIsValid(vmbuffer))
+ {
ReleaseBuffer(vmbuffer);
+ vmbuffer = InvalidBuffer;
+ }
*update_indexes = TU_None;
bms_free(hot_attrs);
@@ -3870,10 +3910,15 @@ l2:
* overhead would be unchanged, that doesn't seem necessarily
* worthwhile.
*/
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear_locked(relation, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ else
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ }
MarkBufferDirty(buffer);
@@ -3892,12 +3937,23 @@ l2:
xlrec.flags =
cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(1, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (cleared_all_frozen)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
/*
@@ -3983,6 +4039,8 @@ l2:
heaptup = newtup;
}
+ newpage = BufferGetPage(newbuf);
+
/*
* We're about to do the actual update -- check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -4096,20 +4154,59 @@ l2:
/* record address of new tuple in t_ctid of old one */
oldtup.t_data->t_ctid = heaptup->t_self;
- /* clear PD_ALL_VISIBLE flags, reset all visibilitymap bits */
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * Clear PD_ALL_VISIBLE flags and reset visibility map bits for any heap
+ * pages that were all-visible. If there are two heap pages, we may need
+ * to clear VM bits for both.
+ */
+ if (PageIsAllVisible(page) &&
+ newbuf != buffer && PageIsAllVisible(newpage) &&
+ vmbuffer_new == vmbuffer)
{
+ /*
+ * This is the more complicated case: both the new and old heap pages
+ * are all-visible and both their VM bits are on the same page of the
+ * VM, so we register a single VM buffer as HEAP_UPDATE_BLKREF_VM_NEW
+ * in the WAL record. We must be careful to only lock and register one
+ * buffer, even though we modify it twice -- once for each heap
+ * block's VM bits.
+ */
+ PageClearAllVisible(page);
+ PageClearAllVisible(newpage);
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear_locked(relation, block,
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear_locked(relation, BufferGetBlockNumber(newbuf),
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS);
+ modified_vmbuffer_new = true;
all_visible_cleared = true;
- PageClearAllVisible(BufferGetPage(buffer));
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ all_visible_cleared_new = true;
}
- if (newbuf != buffer && PageIsAllVisible(BufferGetPage(newbuf)))
+ else
{
- all_visible_cleared_new = true;
- PageClearAllVisible(BufferGetPage(newbuf));
- visibilitymap_clear(relation, BufferGetBlockNumber(newbuf),
- vmbuffer_new, VISIBILITYMAP_VALID_BITS);
+ /*
+ * In all the remaining cases, we will clear at most one heap block's
+ * VM bits per VM page, so we don't have to worry about locking or
+ * registering the same buffer twice.
+ */
+ if (PageIsAllVisible(page))
+ {
+ PageClearAllVisible(page);
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear_locked(relation, block,
+ vmbuffer, VISIBILITYMAP_VALID_BITS);
+ modified_vmbuffer = true;
+ all_visible_cleared = true;
+ }
+ if (newbuf != buffer && PageIsAllVisible(newpage))
+ {
+ PageClearAllVisible(newpage);
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear_locked(relation, BufferGetBlockNumber(newbuf),
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS);
+ modified_vmbuffer_new = true;
+ all_visible_cleared_new = true;
+ }
}
if (newbuf != buffer)
@@ -4132,7 +4229,10 @@ l2:
}
recptr = log_heap_update(relation, buffer,
- newbuf, &oldtup, heaptup,
+ modified_vmbuffer ? vmbuffer : InvalidBuffer,
+ newbuf,
+ modified_vmbuffer_new ? vmbuffer_new : InvalidBuffer,
+ &oldtup, heaptup,
old_key_tuple,
all_visible_cleared,
all_visible_cleared_new);
@@ -4141,10 +4241,20 @@ l2:
PageSetLSN(BufferGetPage(newbuf), recptr);
}
PageSetLSN(BufferGetPage(buffer), recptr);
+
+ if (modified_vmbuffer)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
+ if (modified_vmbuffer_new)
+ PageSetLSN(BufferGetPage(vmbuffer_new), recptr);
}
END_CRIT_SECTION();
+ if (modified_vmbuffer)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ if (modified_vmbuffer_new)
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_UNLOCK);
+
if (newbuf != buffer)
LockBuffer(newbuf, BUFFER_LOCK_UNLOCK);
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
@@ -5193,10 +5303,15 @@ failed:
tuple->t_data->t_ctid = *tid;
/* Clear only the all-frozen bit on visibility map if needed */
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear_locked(relation, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ else
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ }
MarkBufferDirty(*buffer);
@@ -5228,15 +5343,25 @@ failed:
xlrec.flags = cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(1, vmbuffer, 0);
+
/* we don't decode row locks atm, so no need to log the origin */
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (cleared_all_frozen)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
result = TM_Ok;
out_locked:
@@ -5947,10 +6072,15 @@ l4:
xid, mode, false,
&new_xmax, &new_infomask, &new_infomask2);
- if (PageIsAllVisible(BufferGetPage(buf)) &&
- visibilitymap_clear(rel, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(BufferGetPage(buf)))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear_locked(rel, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ else
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ }
START_CRIT_SECTION();
@@ -5981,13 +6111,23 @@ l4:
XLogRegisterData(&xlrec, SizeOfHeapLockUpdated);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(1, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_LOCK_UPDATED);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (cleared_all_frozen)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
next:
/* if we find the end of update chain, we're done. */
if (mytup.t_data->t_infomask & HEAP_XMAX_INVALID ||
@@ -8844,8 +8984,9 @@ log_heap_visible(Relation rel, Buffer heap_buffer, Buffer vm_buffer,
* have modified the buffer(s) and marked them dirty.
*/
static XLogRecPtr
-log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup, HeapTuple newtup,
+log_heap_update(Relation reln, Buffer oldbuf, Buffer vmbuffer_old,
+ Buffer newbuf, Buffer vmbuffer_new,
+ HeapTuple oldtup, HeapTuple newtup,
HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared)
{
@@ -8972,9 +9113,9 @@ log_heap_update(Relation reln, Buffer oldbuf,
if (need_tuple_data)
bufflags |= REGBUF_KEEP_DATA;
- XLogRegisterBuffer(0, newbuf, bufflags);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_NEW, newbuf, bufflags);
if (oldbuf != newbuf)
- XLogRegisterBuffer(1, oldbuf, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_OLD, oldbuf, REGBUF_STANDARD);
XLogRegisterData(&xlrec, SizeOfHeapUpdate);
@@ -9051,6 +9192,20 @@ log_heap_update(Relation reln, Buffer oldbuf,
old_key_tuple->t_len - SizeofHeapTupleHeader);
}
+ /*
+ * Register VM buffers. If the old and new heap pages' VM bits are on the
+ * same VM page, the caller passes only vmbuffer_new (mirroring the heap
+ * page convention where block 0 = new is always registered).
+ */
+ Assert((BufferIsInvalid(vmbuffer_old) && BufferIsInvalid(vmbuffer_new)) ||
+ (vmbuffer_old != vmbuffer_new));
+
+ if (BufferIsValid(vmbuffer_new))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_NEW, vmbuffer_new, 0);
+
+ if (BufferIsValid(vmbuffer_old))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_OLD, vmbuffer_old, 0);
+
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index eb4bd3d6ae3..364e51791a5 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -363,9 +363,24 @@ heap_xlog_delete(XLogReaderState *record)
Relation reln = CreateFakeRelcacheEntry(target_locator);
Buffer vmbuffer = InvalidBuffer;
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
+ if (XLogRecHasBlockRef(record, 1))
+ {
+ if (XLogReadBufferForRedo(record, 1, &vmbuffer) == BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln,
+ blkno, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else
+ {
+ visibilitymap_pin(reln, blkno, &vmbuffer);
+ visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
FreeFakeRelcacheEntry(reln);
}
@@ -450,9 +465,24 @@ heap_xlog_insert(XLogReaderState *record)
Relation reln = CreateFakeRelcacheEntry(target_locator);
Buffer vmbuffer = InvalidBuffer;
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
+ if (XLogRecHasBlockRef(record, 1))
+ {
+ if (XLogReadBufferForRedo(record, 1, &vmbuffer) == BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln,
+ blkno, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else
+ {
+ visibilitymap_pin(reln, blkno, &vmbuffer);
+ visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
FreeFakeRelcacheEntry(reln);
}
@@ -566,17 +596,38 @@ heap_xlog_multi_insert(XLogReaderState *record)
(xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
/*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
+ * If required, clear the VM first, to prevent all-visible temporarily
+ * being set for a heap page that's not all visible anymore.
+ *
+ * If it were possible for XLH_INSERT_ALL_VISIBLE_CLEARED and
+ * XLH_INSERT_ALL_FROZEN_SET to be present in the same record, doing the
+ * XLogReadBufferForRedo() before the PageSetAllVisible() below would be a
+ * problem, as it'd violate the rule that a heap page must never be set
+ * all-visible in the VM while its PD_ALL_VISIBLE is clear.
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
{
Relation reln = CreateFakeRelcacheEntry(rlocator);
Buffer vmbuffer = InvalidBuffer;
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
+ if (XLogRecHasBlockRef(record, 1))
+ {
+ if (XLogReadBufferForRedo(record, 1, &vmbuffer) == BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln,
+ blkno, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else
+ {
+ visibilitymap_pin(reln, blkno, &vmbuffer);
+ visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
FreeFakeRelcacheEntry(reln);
}
@@ -691,6 +742,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Buffer obuffer,
nbuffer;
Page page;
+ bool new_cleared,
+ old_cleared;
OffsetNumber offnum;
ItemId lp = NULL;
HeapTupleData oldtup;
@@ -728,14 +781,107 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
*/
- if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED)
+ new_cleared = (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0;
+ old_cleared = (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) != 0;
+ if (new_cleared || old_cleared)
{
Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
+ bool has_vm_old = XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_OLD);
+ bool has_vm_new = XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_NEW);
+
+ if (has_vm_new || has_vm_old)
+ {
+ /*
+ * If both the old and new heap pages were all-visible and their
+ * VM bits are on the same VM page, that single VM page is
+ * registered as HEAP_UPDATE_BLKREF_VM_NEW. Clear both heap
+ * blocks' VM bits from the single provided VM buffer.
+ */
+ bool same_vm_page = new_cleared && old_cleared &&
+ !has_vm_old && has_vm_new;
+
+ if (same_vm_page)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW, &vmbuffer) ==
+ BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear_locked(reln, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else
+ {
+ /*
+ * We can be sure that we need to clear at most one heap
+ * page's VM bits in each registered VM buffer.
+ */
+ if (new_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW, &vmbuffer) ==
+ BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ if (old_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_OLD, &vmbuffer) ==
+ BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ }
+ }
+ else
+ {
+ /*
+ * Backwards compatibility path. Previously, the VM buffers were
+ * not registered in the WAL record. We need this path to replay
+ * WAL generated by a not-yet-patched primary during upgrade.
+ */
+ if (old_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ visibilitymap_pin(reln, oldblk, &vmbuffer);
+ visibilitymap_clear(reln, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
+ if (new_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ visibilitymap_pin(reln, newblk, &vmbuffer);
+ visibilitymap_clear(reln, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
+ }
- visibilitymap_pin(reln, oldblk, &vmbuffer);
- visibilitymap_clear(reln, oldblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -808,21 +954,6 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
else
newaction = XLogReadBufferForRedo(record, 0, &nbuffer);
- /*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
- */
- if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, newblk, &vmbuffer);
- visibilitymap_clear(reln, newblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
-
/* Deal with new tuple */
if (newaction == BLK_NEEDS_REDO)
{
@@ -1018,10 +1149,23 @@ heap_xlog_lock(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
reln = CreateFakeRelcacheEntry(rlocator);
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
-
- ReleaseBuffer(vmbuffer);
+ if (XLogRecHasBlockRef(record, 1))
+ {
+ if (XLogReadBufferForRedo(record, 1, &vmbuffer) == BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else
+ {
+ visibilitymap_pin(reln, block, &vmbuffer);
+ visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
+ ReleaseBuffer(vmbuffer);
+ }
FreeFakeRelcacheEntry(reln);
}
@@ -1094,10 +1238,23 @@ heap_xlog_lock_updated(XLogReaderState *record)
XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
reln = CreateFakeRelcacheEntry(rlocator);
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
-
- ReleaseBuffer(vmbuffer);
+ if (XLogRecHasBlockRef(record, 1))
+ {
+ if (XLogReadBufferForRedo(record, 1, &vmbuffer) == BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else
+ {
+ visibilitymap_pin(reln, block, &vmbuffer);
+ visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
+ ReleaseBuffer(vmbuffer);
+ }
FreeFakeRelcacheEntry(reln);
}
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 1874a3fda37..89079b52a2f 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -135,9 +135,27 @@ static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks);
* You must pass a buffer containing the correct map page to this function.
* Call visibilitymap_pin first to pin the right one. This function doesn't do
* any I/O. Returns true if any bits have been cleared and false otherwise.
+ *
+ * This is retained for backwards compatability, as it is usually necessary to
+ * register the VM buffer in the WAL record and this necessitates holding the
+ * lock for longer than it is held here.
*/
bool
visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
+{
+ bool cleared = false;
+
+ LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+
+ cleared = visibilitymap_clear_locked(rel, heapBlk, vmbuf, flags);
+
+ LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
+
+ return cleared;
+}
+
+bool
+visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
{
BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
@@ -157,7 +175,8 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
elog(ERROR, "wrong buffer passed to visibilitymap_clear");
- LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+ Assert(BufferIsExclusiveLocked(vmbuf));
+
map = PageGetContents(BufferGetPage(vmbuf));
if (map[mapByte] & mask)
@@ -168,7 +187,6 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
cleared = true;
}
- LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
return cleared;
}
diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl
index 0f98c7df82e..b545098be60 100644
--- a/src/bin/pg_walsummary/t/002_blocks.pl
+++ b/src/bin/pg_walsummary/t/002_blocks.pl
@@ -93,13 +93,14 @@ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
split(m@/@, $end_lsn);
ok(-f $filename, "WAL summary file exists");
-# Run pg_walsummary on it. We expect exactly two blocks to be modified,
-# block 0 and one other.
+# Run pg_walsummary on it. We expect exactly three blocks to be modified,
+# block 0 (old tuple), another block (new tuple) and the block for the VM.
my ($stdout, $stderr) = run_command([ 'pg_walsummary', '-i', $filename ]);
note($stdout);
@lines = split(/\n/, $stdout);
like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified");
+like($stdout, qr/FORK vm: block 0$/m, "stdout shows VM block 0 modified");
is($stderr, '', 'stderr is empty');
-is(0 + @lines, 2, "UPDATE modified 2 blocks");
+is(0 + @lines, 3, "UPDATE modified 3 blocks");
done_testing();
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 277df6b3cf0..0e55ae58b5c 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -214,7 +214,22 @@ typedef struct xl_multi_insert_tuple
* included even if a full-page image was taken.
*
* Backup blk 1: old page, if different. (no data, just a reference to the blk)
+ *
+ * Backup blk 2: VM page covering the new heap page. Registered whenever
+ * XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED is set. Also covers the old
+ * heap page's VM bits when both heap pages map to the same VM
+ * page.
+ *
+ * Backup blk 3: VM page covering the old heap page. Only registered when
+ * XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED is set and the old heap
+ * page's VM bits are on a different VM page from the new heap
+ * page's.
*/
+#define HEAP_UPDATE_BLKREF_NEW 0
+#define HEAP_UPDATE_BLKREF_OLD 1
+#define HEAP_UPDATE_BLKREF_VM_NEW 2
+#define HEAP_UPDATE_BLKREF_VM_OLD 3
+
typedef struct xl_heap_update
{
TransactionId old_xmax; /* xmax of the old tuple */
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index ea889bf9ec7..1bc59c9ac33 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -26,6 +26,8 @@
#define VM_ALL_FROZEN(r, b, v) \
((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0)
+extern bool visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk,
+ Buffer vmbuf, uint8 flags);
extern bool visibilitymap_clear(Relation rel, BlockNumber heapBlk,
Buffer vmbuf, uint8 flags);
extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk,
--
2.43.0
v1_PG18-0002-Turn-off-debug-for-pg_combinebackup-TAP-test.patch (text/x-patch, 1.9K) download
From 7b2f1cd78605cfc7812a41f616d47a8d1b4fed46 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 30 Apr 2026 16:06:06 -0400
Subject: [PATCH v1_PG18 2/3] Turn off debug for pg_combinebackup TAP tests
Having --debug as the default option to pg_combinebackup in
init_from_backup() makes the logs extremely verbose. It seems better to
let developers set an environment variable or pass it as a keyword
argument to init_from_backup() to turn on debugging as they wish.
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 2e31c23a6ac..53ce8169f39 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -898,7 +898,11 @@ pathnames that should be used for the target cluster.
To restore from an incremental backup, pass the parameter combine_with_prior
as a reference to an array of prior backup names with which this backup
-is to be combined using pg_combinebackup.
+is to be combined using pg_combinebackup. By default, pg_combinebackup is
+run without --debug. To enable --debug output when diagnosing test
+failures, either pass combine_debug => 1 as a keyword parameter, or set
+the environment variable PG_TEST_PG_COMBINEBACKUP_DEBUG to a true value
+to enable it globally for all callers.
Streaming replication can be enabled on this node by passing the keyword
parameter has_streaming => 1. This is disabled by default.
@@ -949,7 +953,9 @@ sub init_from_backup
}
local %ENV = $self->_get_env();
- my @combineargs = ('pg_combinebackup', '--debug');
+ my @combineargs = ('pg_combinebackup');
+ push @combineargs, '--debug'
+ if $params{combine_debug} // $ENV{PG_TEST_PG_COMBINEBACKUP_DEBUG};
if (exists $params{tablespace_map})
{
while (my ($olddir, $newdir) = each %{ $params{tablespace_map} })
--
2.43.0
v1_PG18-0003-Test-that-VM-clear-registers-VM-buffers.patch (text/x-patch, 10.1K) download
From 16bd0960f9054c03d41d62590fce71a67e06a1b7 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 23 Apr 2026 11:13:23 -0400
Subject: [PATCH v1_PG18 3/3] Test that VM clear registers VM buffers
WAL summarizer only includes registered buffers, so you could end up
with data corruption in incremental backups if VM clears were meant to
be covered by the incremental backup.
---
src/bin/pg_combinebackup/meson.build | 1 +
.../pg_combinebackup/t/012_vm_consistency.pl | 270 ++++++++++++++++++
2 files changed, 271 insertions(+)
create mode 100644 src/bin/pg_combinebackup/t/012_vm_consistency.pl
diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build
index bbc4c5735ba..757b78e2fe8 100644
--- a/src/bin/pg_combinebackup/meson.build
+++ b/src/bin/pg_combinebackup/meson.build
@@ -39,6 +39,7 @@ tests += {
't/009_no_full_file.pl',
't/010_hardlink.pl',
't/011_ib_truncation.pl',
+ 't/012_vm_consistency.pl',
],
}
}
diff --git a/src/bin/pg_combinebackup/t/012_vm_consistency.pl b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
new file mode 100644
index 00000000000..cb03f40bfea
--- /dev/null
+++ b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
@@ -0,0 +1,270 @@
+# Copyright (c) 2021-2026, PostgreSQL Global Development Group
+#
+# Test that heap operations clearing visibility map bits (INSERT, UPDATE,
+# DELETE, SELECT FOR UPDATE, COPY) are correctly registering visibility map
+# buffers, since incremental backups rely on WAL summarizer which only tracks
+# registered buffers.
+#
+# Each operation is tested in a separate vacuum-modify-backup-validate
+# cycle to isolate failures.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir_short();
+my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
+
+# Set up primary with WAL summarization enabled.
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(has_archiving => 1, allows_streaming => 1);
+$primary->append_conf('postgresql.conf', <<EOF);
+summarize_wal = on
+autovacuum = off
+EOF
+$primary->start;
+
+$primary->safe_psql('postgres', q{CREATE EXTENSION pg_visibility});
+
+# Create tables for each test case.
+$primary->safe_psql('postgres', <<SQL);
+CREATE TABLE vm_insert_test (id int);
+INSERT INTO vm_insert_test DEFAULT VALUES;
+
+CREATE TABLE vm_delete_test (id int);
+INSERT INTO vm_delete_test VALUES (1), (2);
+
+CREATE TABLE vm_update_test (id INT, val TEXT);
+INSERT INTO vm_update_test SELECT 1, 'same page';
+INSERT INTO vm_update_test SELECT i, repeat('a', 200) FROM generate_series(2, 70) i;
+
+CREATE TABLE vm_lock_test (id int);
+INSERT INTO vm_lock_test VALUES (1), (2);
+
+CREATE TABLE vm_copy_test (id int);
+INSERT INTO vm_copy_test DEFAULT VALUES;
+SQL
+
+my $backup_num = 0;
+my $restore_num = 0;
+
+# Helper: get both all_visible and all_frozen as a two-element list.
+sub get_vm_summary
+{
+ my ($node, $table) = @_;
+ my $result = $node->safe_psql('postgres',
+ "SELECT all_visible, all_frozen FROM pg_visibility_map_summary('$table')");
+ my @vals = split(/\|/, $result);
+ return @vals;
+}
+
+# Helper: take a full backup. Returns ($path, $name).
+sub take_full_backup
+{
+ $backup_num++;
+ my $name = "full_$backup_num";
+ my $path = $primary->backup_dir . "/$name";
+ $primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $path,
+ '--checkpoint' => 'fast',
+ ],
+ "full backup $backup_num");
+ return ($path, $name);
+}
+
+# Helper: take an incremental backup. Returns ($path, $name).
+sub take_incr_backup
+{
+ my ($prior_path) = @_;
+ $backup_num++;
+ my $name = "incr_$backup_num";
+ my $path = $primary->backup_dir . "/$name";
+ $primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $path,
+ '--checkpoint' => 'fast',
+ '--incremental' => $prior_path . '/backup_manifest',
+ ],
+ "incremental backup $backup_num");
+ return ($path, $name);
+}
+
+# Helper: combine two backups, restore, validate VM for the given table,
+# then stop the restored server. Checks both all_visible and all_frozen.
+sub validate_restored_vm
+{
+ my ($full_path, $full_name, $incr_path, $incr_name, $table, $label) = @_;
+
+ $restore_num++;
+
+ my $restored = PostgreSQL::Test::Cluster->new("restored_$restore_num");
+ $restored->init_from_backup($primary, $incr_name,
+ combine_with_prior => [$full_name],
+ combine_mode => $mode);
+ $restored->append_conf('postgresql.conf', <<EOF);
+autovacuum = off
+EOF
+ $restored->start;
+ $restored->safe_psql('postgres',
+ q{CREATE EXTENSION IF NOT EXISTS pg_visibility});
+
+ my ($primary_visible, $primary_frozen) = get_vm_summary($primary, $table);
+ my ($restored_visible, $restored_frozen) = get_vm_summary($restored, $table);
+
+ is($restored_visible, $primary_visible,
+ "$label: restored all_visible count matches primary");
+ is($restored_frozen, $primary_frozen,
+ "$label: restored all_frozen count matches primary");
+
+ my $corrupt_tids = $restored->safe_psql('postgres',
+ "SELECT count(*) FROM pg_check_visible('$table')");
+ is($corrupt_tids, '0',
+ "$label: no VM corruption detected by pg_check_visible");
+
+ $restored->stop;
+}
+
+##
+## Test 1: INSERT clears all-visible and all-frozen bits
+##
+
+$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_insert_test});
+my ($pre_ins_vis, $pre_ins_frz) = get_vm_summary($primary, 'vm_insert_test');
+cmp_ok($pre_ins_vis, '>', 0,
+ "INSERT test: pages are all-visible after vacuum");
+cmp_ok($pre_ins_frz, '>', 0,
+ "INSERT test: pages are all-frozen after vacuum");
+
+my ($full1, $full1_name) = take_full_backup();
+
+$primary->safe_psql('postgres',
+ q{INSERT INTO vm_insert_test VALUES (1)});
+
+my ($post_ins_vis, $post_ins_frz) = get_vm_summary($primary, 'vm_insert_test');
+cmp_ok($post_ins_vis, '<', $pre_ins_vis,
+ "INSERT test: all-visible cleared after insert");
+cmp_ok($post_ins_frz, '<', $pre_ins_frz,
+ "INSERT test: all-frozen cleared after insert");
+
+my ($incr1, $incr1_name) = take_incr_backup($full1);
+validate_restored_vm($full1, $full1_name, $incr1, $incr1_name,
+ 'vm_insert_test', 'INSERT test');
+
+##
+## Test 2: DELETE clears all-visible and all-frozen bits
+##
+
+$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_delete_test});
+my ($pre_del_vis, $pre_del_frz) = get_vm_summary($primary, 'vm_delete_test');
+cmp_ok($pre_del_vis, '>', 0,
+ "DELETE test: pages are all-visible after vacuum");
+cmp_ok($pre_del_frz, '>', 0,
+ "DELETE test: pages are all-frozen after vacuum");
+
+my ($full2, $full2_name) = take_full_backup();
+
+$primary->safe_psql('postgres',
+ q{DELETE FROM vm_delete_test WHERE id = 1});
+
+my ($post_del_vis, $post_del_frz) = get_vm_summary($primary, 'vm_delete_test');
+cmp_ok($post_del_vis, '<', $pre_del_vis,
+ "DELETE test: all-visible cleared after delete");
+cmp_ok($post_del_frz, '<', $pre_del_frz,
+ "DELETE test: all-frozen cleared after delete");
+
+my ($incr2, $incr2_name) = take_incr_backup($full2);
+validate_restored_vm($full2, $full2_name, $incr2, $incr2_name,
+ 'vm_delete_test', 'DELETE test');
+
+##
+## Test 3: UPDATE clears all-visible and all-frozen bits
+##
+
+$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_update_test});
+my ($pre_upd_vis, $pre_upd_frz) = get_vm_summary($primary, 'vm_update_test');
+cmp_ok($pre_upd_vis, '>', 0,
+ "UPDATE test: pages are all-visible after vacuum");
+cmp_ok($pre_upd_frz, '>', 0,
+ "UPDATE test: pages are all-frozen after vacuum");
+
+my ($full3, $full3_name) = take_full_backup();
+
+# This should update the same page
+$primary->safe_psql('postgres',
+ q{UPDATE vm_update_test SET id = 0 WHERE id = 1});
+# At least some of these updates should be cross-page
+$primary->safe_psql('postgres',
+ q{UPDATE vm_update_test SET id = 99 WHERE id > 50});
+
+my ($post_upd_vis, $post_upd_frz) = get_vm_summary($primary, 'vm_update_test');
+cmp_ok($post_upd_vis, '<', $pre_upd_vis,
+ "UPDATE test: all-visible cleared after update");
+cmp_ok($post_upd_frz, '<', $pre_upd_frz,
+ "UPDATE test: all-frozen cleared after update");
+
+my ($incr3, $incr3_name) = take_incr_backup($full3);
+validate_restored_vm($full3, $full3_name, $incr3, $incr3_name,
+ 'vm_update_test', 'UPDATE test');
+
+##
+## Test 4: SELECT FOR UPDATE clears all-frozen bit (but not all-visible)
+##
+## heap_lock_tuple only clears VISIBILITYMAP_ALL_FROZEN, not ALL_VISIBLE
+##
+
+$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_lock_test});
+my ($pre_lock_vis, $pre_lock_frz) = get_vm_summary($primary, 'vm_lock_test');
+cmp_ok($pre_lock_vis, '>', 0,
+ "LOCK test: pages are all-visible after vacuum");
+cmp_ok($pre_lock_frz, '>', 0,
+ "LOCK test: pages are all-frozen after vacuum");
+
+my ($full4, $full4_name) = take_full_backup();
+
+$primary->safe_psql('postgres',
+ q{SELECT * FROM vm_lock_test WHERE id = 1 FOR UPDATE});
+
+my ($post_lock_vis, $post_lock_frz) = get_vm_summary($primary, 'vm_lock_test');
+is($post_lock_vis, $pre_lock_vis,
+ "LOCK test: all-visible unchanged after SELECT FOR UPDATE");
+cmp_ok($post_lock_frz, '<', $pre_lock_frz,
+ "LOCK test: all-frozen cleared after SELECT FOR UPDATE");
+
+my ($incr4, $incr4_name) = take_incr_backup($full4);
+validate_restored_vm($full4, $full4_name, $incr4, $incr4_name,
+ 'vm_lock_test', 'LOCK test');
+
+##
+## Test 5: COPY clears all-visible and all-frozen bits
+##
+
+$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_copy_test});
+my ($pre_copy_vis, $pre_copy_frz) = get_vm_summary($primary, 'vm_copy_test');
+cmp_ok($pre_copy_vis, '>', 0,
+ "COPY test: pages are all-visible after vacuum");
+cmp_ok($pre_copy_frz, '>', 0,
+ "COPY test: pages are all-frozen after vacuum");
+
+my ($full5, $full5_name) = take_full_backup();
+
+$primary->safe_psql('postgres',
+ "COPY vm_copy_test FROM PROGRAM 'echo 42'");
+
+my ($post_copy_vis, $post_copy_frz) = get_vm_summary($primary, 'vm_copy_test');
+cmp_ok($post_copy_vis, '<', $pre_copy_vis,
+ "COPY test: all-visible cleared after COPY");
+cmp_ok($post_copy_frz, '<', $pre_copy_frz,
+ "COPY test: all-frozen cleared after COPY");
+
+my ($incr5, $incr5_name) = take_incr_backup($full5);
+validate_restored_vm($full5, $full5_name, $incr5, $incr5_name,
+ 'vm_copy_test', 'COPY test');
+
+$primary->stop;
+
+done_testing();
--
2.43.0
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-05-01 16:41 Robert Haas <robertmhaas@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
2 siblings, 1 reply; 57+ messages in thread
From: Robert Haas @ 2026-05-01 16:41 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Thu, Apr 30, 2026 at 5:44 PM Melanie Plageman
<melanieplageman@gmail.com> wrote:
> I added a TAP test that does something like Andres' incremental backup
> repro but tests more variations. Because init_from_backup() runs
> pg_combinebackup with the --debug flag, the log output is very verbose
> (i.e. every single copied file is logged). I suggest we turn it off by
> default in tests. I included a patch that does that. It isn't required
> to commit my test, but my test does produce 5000 lines of regression
> log output which seems...not ideal.
In my experience, it's pretty much impossible to debug a pg_basebackup
failure with incremental backup without the --debug option. Like,
you'll just have no idea where things went wrong. So I am cautious
about this, but I also understand that we run the tests a lot of times
on a lot of branches on a lot of machines, and so 5000 lines of output
that seems like no big deal in isolation may start to add up to a big
deal when you multiply it out.
So I feel like if your reasoning here is "this is using too many
resources and we can't afford it," that's fair enough, and we'll just
have to accept that if it starts failing we'll have to turn this back
on to have any idea what the problem is. If your reasoning is more
like "well I don't want all this output," I don't really think that's
a great reason -- if it ever fails, you will.
Another possible approach could be to direct the output to a temporary
file. At the end of the test run, if no tests failed, remove the file.
If there were any failures, copy that file's contents into the log.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-05-01 16:58 Andres Freund <andres@anarazel.de>
parent: Robert Haas <robertmhaas@gmail.com>
0 siblings, 1 reply; 57+ messages in thread
From: Andres Freund @ 2026-05-01 16:58 UTC (permalink / raw)
To: Robert Haas <robertmhaas@gmail.com>; +Cc: Melanie Plageman <melanieplageman@gmail.com>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
Hi,
On 2026-05-01 12:41:49 -0400, Robert Haas wrote:
> On Thu, Apr 30, 2026 at 5:44 PM Melanie Plageman
> <melanieplageman@gmail.com> wrote:
> > I added a TAP test that does something like Andres' incremental backup
> > repro but tests more variations. Because init_from_backup() runs
> > pg_combinebackup with the --debug flag, the log output is very verbose
> > (i.e. every single copied file is logged). I suggest we turn it off by
> > default in tests. I included a patch that does that. It isn't required
> > to commit my test, but my test does produce 5000 lines of regression
> > log output which seems...not ideal.
>
> In my experience, it's pretty much impossible to debug a pg_basebackup
> failure with incremental backup without the --debug option. Like,
> you'll just have no idea where things went wrong. So I am cautious
> about this, but I also understand that we run the tests a lot of times
> on a lot of branches on a lot of machines, and so 5000 lines of output
> that seems like no big deal in isolation may start to add up to a big
> deal when you multiply it out.
>
> So I feel like if your reasoning here is "this is using too many
> resources and we can't afford it," that's fair enough, and we'll just
> have to accept that if it starts failing we'll have to turn this back
> on to have any idea what the problem is. If your reasoning is more
> like "well I don't want all this output," I don't really think that's
> a great reason -- if it ever fails, you will.
Well, right now the regress_log* files that involve combining backups are so
long that it's hard to find the actual stuff that's happening - and it's worse
with the test Melanie is proposing because it runs a few iterations of
combining backups. So I think it's not universal that you'd want it.
> Another possible approach could be to direct the output to a temporary
> file. At the end of the test run, if no tests failed, remove the file.
> If there were any failures, copy that file's contents into the log.
I think that's probably the better direction. I'd not even include it
directly into regress_log, just put it alongside it, like we have the server
logs etc.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-05-01 17:30 Robert Haas <robertmhaas@gmail.com>
parent: Andres Freund <andres@anarazel.de>
0 siblings, 0 replies; 57+ messages in thread
From: Robert Haas @ 2026-05-01 17:30 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; +Cc: Melanie Plageman <melanieplageman@gmail.com>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Fri, May 1, 2026 at 12:58 PM Andres Freund <andres@anarazel.de> wrote:
> Well, right now the regress_log* files that involve combining backups are so
> long that it's hard to find the actual stuff that's happening - and it's worse
> with the test Melanie is proposing because it runs a few iterations of
> combining backups. So I think it's not universal that you'd want it.
Fair enough.
> > Another possible approach could be to direct the output to a temporary
> > file. At the end of the test run, if no tests failed, remove the file.
> > If there were any failures, copy that file's contents into the log.
>
> I think that's probably the better direction. I'd not even include it
> directly into regress_log, just put it alongside it, like we have the server
> logs etc.
Oh, yeah, OK, as long as CI will capture it, that WFM.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-05-01 19:14 Andres Freund <andres@anarazel.de>
parent: Melanie Plageman <melanieplageman@gmail.com>
2 siblings, 1 reply; 57+ messages in thread
From: Andres Freund @ 2026-05-01 19:14 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
Hi,
On 2026-04-30 17:44:28 -0400, Melanie Plageman wrote:
> On Thu, Mar 5, 2026 at 4:01 PM Andres Freund <andres@anarazel.de> wrote:
> >
> > On 2026-03-05 15:38:24 -0500, Andres Freund wrote:
> >
> > > There's explicit code for ignoring the FSM, but I don't see the same for the
> > > VM. And that makes sense: VM changes are mostly WAL logged, just not
> > > completely / generically (i.e. this complaint), whereas FSM changes are not
> > > WAL logged at all.
> >
> > Unfortunately I can confirm that incremental backups end up with an outdated
> > VM.
> >
> > An unfortunate kicker: It looks like verify_heapam() doesn't even notice :(.
>
> Attached is a patch set to fix the issue based largely on the work you
> started on your branch. I attached the version targeting master/19
> which is prefixed with v1_PGMASTER and the version targeting 18,
> prefixed v1_PG18. The pg 18 changes aren't a straight cherry-pick to
> 17 (the earliest I'll backpatch because that was when incremental
> backup was introduced) because the redo functions live in a different
> file in 18 than in 17, but I want to avoid discussing three different
> versions of this patch set on this thread.
Makes sense.
I think it's worth explicitly bringing up that this means that we leave an
arguable corruption case in the earlier branches.
I think it's the right call: The changes are non-trivial and there are
plenty-inter branch changes. And I can't come up with realistic scenarios
where it'd be a problem without something like incremental backup (as
pg_rewind just always copies the VM).
> I added a TAP test that does something like Andres' incremental backup
> repro but tests more variations. Because init_from_backup() runs
> pg_combinebackup with the --debug flag, the log output is very verbose
> (i.e. every single copied file is logged). I suggest we turn it off by
> default in tests. I included a patch that does that. It isn't required
> to commit my test, but my test does produce 5000 lines of regression
> log output which seems...not ideal.
>
> The patch set targeting master includes a few more changes aimed at
> catching bugs like this in the future. 0005 adds a check to
> verify_heapam() that PD_ALL_VISIBLE is never clear when the VM is set.
> 0006 stops masking PD_ALL_VISIBLE during WAL consistency checking. And
> 0007 is a version of a patch Andres started which validates that every
> block registered with a WAL record is read during recovery.
I thought I'd start looking at the 18 patches.
> From 8ef81094e39a34d845b1a40cad7de469c8501bee Mon Sep 17 00:00:00 2001
> From: Melanie Plageman <melanieplageman@gmail.com>
> Date: Wed, 29 Apr 2026 12:53:49 -0400
> Subject: [PATCH v1_PG18 1/3] Fix WAL logging of VM clears
>
> Previously, we failed to register the visibility map buffer when
> emitting WAL after clearing the VM bits for heap pages (recovery code
> read the VM page normally). This meant that we couldn't emit FPIs of VM
> pages when clearing VM bits. While this would not result in a checksum
> error because the visibility map is read with RBM_ZERO_ON_ERROR, WAL
> summarizer only includes pages which were registered, so a restore from
> an incremental backup would not include the modified VM pages -- leading
> to data corruption.
>
> Fix this by registering the VM buffer in the WAL record when clearing VM
> bits. This also means the VM buffer must be locked throughout the
> duration of the critical section where we make changes to the VM and
> heap page and emit WAL.
> diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c
> index 3e86283beb7..d7e4e051fff 100644
> --- a/contrib/pg_surgery/heap_surgery.c
> +++ b/contrib/pg_surgery/heap_surgery.c
> @@ -266,9 +266,10 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
> */
> if (PageIsAllVisible(page))
> {
> + LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
> PageClearAllVisible(page);
> - visibilitymap_clear(rel, blkno, vmbuf,
> - VISIBILITYMAP_VALID_BITS);
> + visibilitymap_clear_locked(rel, blkno, vmbuf,
> + VISIBILITYMAP_VALID_BITS);
> did_modify_vm = true;
> }
> }
> @@ -331,7 +332,10 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
>
> UnlockReleaseBuffer(buf);
>
> - if (vmbuf != InvalidBuffer)
> +
> + if (did_modify_vm)
> + UnlockReleaseBuffer(vmbuf);
> + else if (BufferIsValid(vmbuf))
> ReleaseBuffer(vmbuf);
>
> /* Update the current_start_ptr before moving to the next page. */
Why did you touch heap_surgery in 18? That shouldn't be necessary, right?
> @@ -2135,10 +2136,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
> if (PageIsAllVisible(BufferGetPage(buffer)))
> {
> all_visible_cleared = true;
> +
> + LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
> PageClearAllVisible(BufferGetPage(buffer));
> - visibilitymap_clear(relation,
> - ItemPointerGetBlockNumber(&(heaptup->t_self)),
> - vmbuffer, VISIBILITYMAP_VALID_BITS);
> + visibilitymap_clear_locked(relation,
> + ItemPointerGetBlockNumber(&(heaptup->t_self)),
> + vmbuffer, VISIBILITYMAP_VALID_BITS);
> }
>
> /*
It probably doesn't matter, but it seems like it'd be ever so slightly more
correct to first do visibilitymap_clear_locked() and then
PageClearAllVisible(), given
a) the rule that there should never be a VM bit set when !PD_ALL_VISIBLE
b) VM reads happening without page locks
I don't think it can ever lead to a wrong result, but still seems better to do
it the "right" way round.
> @@ -2228,13 +2231,22 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
> /* filtering by origin on a row level is much more efficient */
> XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
>
> + if (all_visible_cleared)
> + XLogRegisterBuffer(1, vmbuffer, 0);
> +
For update you created HEAP_UPDATE_BLKREF_* to distinguish the different block
refernces. I think that was a good call to make the code at least somewhat
understandable. I think once we do it for heap_update(), we should just do it
for heap_{insert,multi_insert,delete,update} too.
> @@ -2503,10 +2515,11 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
> if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
> {
> all_visible_cleared = true;
> + LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
> PageClearAllVisible(page);
> - visibilitymap_clear(relation,
> - BufferGetBlockNumber(buffer),
> - vmbuffer, VISIBILITYMAP_VALID_BITS);
> + visibilitymap_clear_locked(relation,
> + BufferGetBlockNumber(buffer),
> + vmbuffer, VISIBILITYMAP_VALID_BITS);
> }
Every time I see one of these new lock acquisitions I get worried about lock
ordering issues leading to deadlocks. Wrongly here, because
visibilitymap_clear() would also have acquired the lock.
Am I blind, or have not documented the lock ordering rules for heap vs VM
anywhere?
> @@ -3133,13 +3154,23 @@ l1:
> /* filtering by origin on a row level is much more efficient */
> XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
>
> + if (all_visible_cleared)
> + XLogRegisterBuffer(1, vmbuffer, 0);
> +
> recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE);
On the topic of complaining about things you have no responsibility for
whatsoever: It's so annoying that git picks up goto labels as diff "subjects"
(the "l1:" above) ...
> + /*
> + * Clear PD_ALL_VISIBLE flags and reset visibility map bits for any heap
> + * pages that were all-visible. If there are two heap pages, we may need
> + * to clear VM bits for both.
> + */
> + if (PageIsAllVisible(page) &&
> + newbuf != buffer && PageIsAllVisible(newpage) &&
> + vmbuffer_new == vmbuffer)
> {
> + /*
> + * This is the more complicated case: both the new and old heap pages
> + * are all-visible and both their VM bits are on the same page of the
> + * VM, so we register a single VM buffer as HEAP_UPDATE_BLKREF_VM_NEW
> + * in the WAL record. We must be careful to only lock and register one
> + * buffer, even though we modify it twice -- once for each heap
> + * block's VM bits.
> + */
> - if (newbuf != buffer && PageIsAllVisible(BufferGetPage(newbuf)))
> + else
> {
> - all_visible_cleared_new = true;
> - PageClearAllVisible(BufferGetPage(newbuf));
> - visibilitymap_clear(relation, BufferGetBlockNumber(newbuf),
> - vmbuffer_new, VISIBILITYMAP_VALID_BITS);
> + /*
> + * In all the remaining cases, we will clear at most one heap block's
> + * VM bits per VM page, so we don't have to worry about locking or
> + * registering the same buffer twice.
> + */
> + if (PageIsAllVisible(page))
> + {
> + PageClearAllVisible(page);
> + LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
> + visibilitymap_clear_locked(relation, block,
> + vmbuffer, VISIBILITYMAP_VALID_BITS);
> + modified_vmbuffer = true;
> + all_visible_cleared = true;
> + }
> + if (newbuf != buffer && PageIsAllVisible(newpage))
> + {
> + PageClearAllVisible(newpage);
> + LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE);
> + visibilitymap_clear_locked(relation, BufferGetBlockNumber(newbuf),
> + vmbuffer_new, VISIBILITYMAP_VALID_BITS);
> + modified_vmbuffer_new = true;
> + all_visible_cleared_new = true;
> + }
I think here we actually have a lock ordering issue:
Assuming that we have four all-visible heap pages, Block HA1, HA2, which are
covered by VM1 and HB1 and HB2 which are covered by VM2. You could get into
the unlucky situation that you have one backend B1 updating a tuple from HA1
and placing it on HB1, and a backend B2 updating a tuple on HB2 and updating
it to HA2.
In that case you could have the following lock acquisition order:
B1: lock HA1
B1: lock HB1
B2: lock HA2
B2: lock HB2
(hio.c will make sure to acquire the locks in ascending page order)
B1: lock VM1
B2: lock VM2
B1: lock VM2 <waits>
B2: lock VM1 <undetected deadlock>
It might not be possible to reach this, due to the newtupsize > pagefree case
clearing the VM bit. But I think you could get unlucky and have it get set
again. For that I think the target page would have to be empty, otherwise we'd
not have set it as all-visible with a pin, but that seems possible.
Coming back later: The newtupsize > pagefree case doesn't even help, because
inexplicably we are just unsetting all frozen, even though we're about to
update the page! WHYYY!?!
So I think we need to be careful and acquire the exlusive locks on the two vm
pages in ascending block number order. That'd turn the above into:
B1: lock VM1
B2: lock VM1 <waits>
B1: lock VM2
B1: xloginsert etc
B1: release locks
B2: lock VM1 <completes>
B2: lock VM2
We just need to do the lock acquisition that way, not the setting and clearing
of the bits. So it's perhaps going to not be *too* ugly.
> @@ -363,9 +363,24 @@ heap_xlog_delete(XLogReaderState *record)
> Relation reln = CreateFakeRelcacheEntry(target_locator);
> Buffer vmbuffer = InvalidBuffer;
>
> - visibilitymap_pin(reln, blkno, &vmbuffer);
> - visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
> - ReleaseBuffer(vmbuffer);
> + if (XLogRecHasBlockRef(record, 1))
> + {
> + if (XLogReadBufferForRedo(record, 1, &vmbuffer) == BLK_NEEDS_REDO)
> + {
> + visibilitymap_clear_locked(reln,
> + blkno, vmbuffer,
> + VISIBILITYMAP_VALID_BITS);
> + PageSetLSN(BufferGetPage(vmbuffer), lsn);
> + }
> + if (BufferIsValid(vmbuffer))
> + UnlockReleaseBuffer(vmbuffer);
> + }
> + else
> + {
> + visibilitymap_pin(reln, blkno, &vmbuffer);
> + visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
> + ReleaseBuffer(vmbuffer);
> + }
> FreeFakeRelcacheEntry(reln);
> }
I think we need a comment explaining why we need this if
(XLogRecHasBlockRef()) dance. Otherwise whoever has to look at this for some
bug in the future will be rather confused.
Perhaps just have one comment, with the other places referring to it? Might
also be sane to leave a breadcrumb on the WAL logging side, that it used to
look different in the same major version.
Wonder if we could package this into some kind of helper, there are a bunch of
copies of this now. But there's enough small differences that it's not
trivial.
>
> @@ -691,6 +742,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
> Buffer obuffer,
> nbuffer;
> Page page;
> + bool new_cleared,
> + old_cleared;
> OffsetNumber offnum;
> ItemId lp = NULL;
> HeapTupleData oldtup;
> @@ -728,14 +781,107 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
There's this comment nearby:
/*
* In normal operation, it is important to lock the two pages in
* page-number order, to avoid possible deadlocks against other update
* operations going the other way. However, during WAL replay there can
* be no other update happening, so we don't need to worry about that. But
* we *do* need to worry that we don't expose an inconsistent state to Hot
* Standby queries --- so the original page can't be unlocked before we've
* added the new tuple to the new page.
*/
I think it's not wrong, but man, that seems like a fragile assumption.
> * The visibility map may need to be fixed even if the heap page is
> * already up-to-date.
> */
> - if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED)
> + new_cleared = (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0;
> + old_cleared = (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) != 0;
> + if (new_cleared || old_cleared)
> {
> Relation reln = CreateFakeRelcacheEntry(rlocator);
> - Buffer vmbuffer = InvalidBuffer;
> + bool has_vm_old = XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_OLD);
> + bool has_vm_new = XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_NEW);
> +
> + if (has_vm_new || has_vm_old)
> + {
> + /*
> + * If both the old and new heap pages were all-visible and their
> + * VM bits are on the same VM page, that single VM page is
> + * registered as HEAP_UPDATE_BLKREF_VM_NEW. Clear both heap
> + * blocks' VM bits from the single provided VM buffer.
> + */
> + bool same_vm_page = new_cleared && old_cleared &&
> + !has_vm_old && has_vm_new;
> +
> + if (same_vm_page)
> + {
> + Buffer vmbuffer = InvalidBuffer;
> +
> + if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW, &vmbuffer) ==
> + BLK_NEEDS_REDO)
> + {
> + visibilitymap_clear_locked(reln, oldblk, vmbuffer,
> + VISIBILITYMAP_VALID_BITS);
> + visibilitymap_clear_locked(reln, newblk, vmbuffer,
> + VISIBILITYMAP_VALID_BITS);
> + PageSetLSN(BufferGetPage(vmbuffer), lsn);
> + }
> +
> + if (BufferIsValid(vmbuffer))
> + UnlockReleaseBuffer(vmbuffer);
> + }
> + else
> + {
> + /*
> + * We can be sure that we need to clear at most one heap
> + * page's VM bits in each registered VM buffer.
> + */
> + if (new_cleared)
> + {
> + Buffer vmbuffer = InvalidBuffer;
> +
> + if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW, &vmbuffer) ==
> + BLK_NEEDS_REDO)
> + {
> + visibilitymap_clear_locked(reln, newblk, vmbuffer,
> + VISIBILITYMAP_VALID_BITS);
> + PageSetLSN(BufferGetPage(vmbuffer), lsn);
> + }
> +
> + if (BufferIsValid(vmbuffer))
> + UnlockReleaseBuffer(vmbuffer);
> + }
> + if (old_cleared)
> + {
> + Buffer vmbuffer = InvalidBuffer;
> +
> + if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_OLD, &vmbuffer) ==
> + BLK_NEEDS_REDO)
> + {
> + visibilitymap_clear_locked(reln, oldblk, vmbuffer,
> + VISIBILITYMAP_VALID_BITS);
> + PageSetLSN(BufferGetPage(vmbuffer), lsn);
> + }
> +
> + if (BufferIsValid(vmbuffer))
> + UnlockReleaseBuffer(vmbuffer);
> + }
> + }
Hm. Could this be simplified to something like:
if (has_vm_new)
{
Buffer vmbuffer = InvalidBuffer;
if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW, &vmbuffer) ==
BLK_NEEDS_REDO)
{
if (new_cleared)
visibilitymap_clear_locked(reln, newblk, vmbuffer,
VISIBILITYMAP_VALID_BITS);
if (old_cleared && !has_vm_old)
visibilitymap_clear_locked(reln, oldblk, vmbuffer,
VISIBILITYMAP_VALID_BITS);
PageSetLSN(BufferGetPage(vmbuffer), lsn);
}
if (BufferIsValid(vmbuffer))
UnlockReleaseBuffer(vmbuffer);
}
if (has_vm_old)
{
Buffer vmbuffer = InvalidBuffer;
Assert(old_cleared);
if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_OLD, &vmbuffer) ==
BLK_NEEDS_REDO)
{
visibilitymap_clear_locked(reln, oldblk, vmbuffer,
VISIBILITYMAP_VALID_BITS);
PageSetLSN(BufferGetPage(vmbuffer), lsn);
}
if (BufferIsValid(vmbuffer))
UnlockReleaseBuffer(vmbuffer);
}
if (!has_vm_new && !has_vm_old)
{
...
Would it make sense to have an assert like
Assert(has_vm_new || !has_vm_old); /* should never just have old VM buffer */
> +bool
> +visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
Given this is a new API, I wonder if we shouldn't already start using the new
signature. If there's any extensions using it (I think there is at least one),
this would just require them to have an ifdef for <17, 17-18, and >= 19.
> +# Set up primary with WAL summarization enabled.
> +my $primary = PostgreSQL::Test::Cluster->new('primary');
> +$primary->init(has_archiving => 1, allows_streaming => 1);
> +$primary->append_conf('postgresql.conf', <<EOF);
> +summarize_wal = on
> +autovacuum = off
> +EOF
A here doc with a closing paren is correct perl? You learn something every
day.
Not particularly pretty imo, but whatever, it's perl.
> +##
> +## Test 1: INSERT clears all-visible and all-frozen bits
> +##
> +
> +$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_insert_test});
> +my ($pre_ins_vis, $pre_ins_frz) = get_vm_summary($primary, 'vm_insert_test');
> +cmp_ok($pre_ins_vis, '>', 0,
> + "INSERT test: pages are all-visible after vacuum");
> +cmp_ok($pre_ins_frz, '>', 0,
> + "INSERT test: pages are all-frozen after vacuum");
> +
> +my ($full1, $full1_name) = take_full_backup();
> +
> +$primary->safe_psql('postgres',
> + q{INSERT INTO vm_insert_test VALUES (1)});
> +
> +my ($post_ins_vis, $post_ins_frz) = get_vm_summary($primary, 'vm_insert_test');
> +cmp_ok($post_ins_vis, '<', $pre_ins_vis,
> + "INSERT test: all-visible cleared after insert");
> +cmp_ok($post_ins_frz, '<', $pre_ins_frz,
> + "INSERT test: all-frozen cleared after insert");
> +
> +my ($incr1, $incr1_name) = take_incr_backup($full1);
> +validate_restored_vm($full1, $full1_name, $incr1, $incr1_name,
> + 'vm_insert_test', 'INSERT test');
> +
> +##
> +## Test 2: DELETE clears all-visible and all-frozen bits
> +##
> +
> +$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_delete_test});
> +my ($pre_del_vis, $pre_del_frz) = get_vm_summary($primary, 'vm_delete_test');
> +cmp_ok($pre_del_vis, '>', 0,
> + "DELETE test: pages are all-visible after vacuum");
> +cmp_ok($pre_del_frz, '>', 0,
> + "DELETE test: pages are all-frozen after vacuum");
> +
> +my ($full2, $full2_name) = take_full_backup();
> +
> +$primary->safe_psql('postgres',
> + q{DELETE FROM vm_delete_test WHERE id = 1});
> +
> +my ($post_del_vis, $post_del_frz) = get_vm_summary($primary, 'vm_delete_test');
> +cmp_ok($post_del_vis, '<', $pre_del_vis,
> + "DELETE test: all-visible cleared after delete");
> +cmp_ok($post_del_frz, '<', $pre_del_frz,
> + "DELETE test: all-frozen cleared after delete");
> +
> +my ($incr2, $incr2_name) = take_incr_backup($full2);
> +validate_restored_vm($full2, $full2_name, $incr2, $incr2_name,
> + 'vm_delete_test', 'DELETE test');
Hm. Since these are looking at different tables, can't we do that with one
backup + incremental backup + combine + validate cycle for all of them?
Also seems like the modifications, tests and messages for all of them could be
moved to a helper?
I assume you verified that the tests catch the problem without the fix?
> From bb4c03a887f8ec5395ce37c09279a0786712b06c Mon Sep 17 00:00:00 2001
> From: Melanie Plageman <melanieplageman@gmail.com>
> Date: Wed, 29 Apr 2026 10:20:20 -0400
> Subject: [PATCH v1_PGMASTER 5/7] Add VM corruption check to verify_heapam()
>
> pg_check_visible() can catch cases were tuples are not visible to all
> and the VM is set, but it doesn't specifically check page hints. This
> doesn't check, for example, an INSERT failing to clear PD_ALL_VISIBLE.
>
> This also helps users who have access to amcheck but not pg_visibility
> to diagnose corruption.
>
> Author: Melanie Plageman <melanieplageman@gmail.com>
> ---
> contrib/amcheck/verify_heapam.c | 16 ++++++++++++++++
> 1 file changed, 16 insertions(+)
>
> diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
> index 20ff58aa782..0e023871566 100644
> --- a/contrib/amcheck/verify_heapam.c
> +++ b/contrib/amcheck/verify_heapam.c
> @@ -481,6 +481,7 @@ verify_heapam(PG_FUNCTION_ARGS)
>
> while ((ctx.buffer = read_stream_next_buffer(stream, NULL)) != InvalidBuffer)
> {
> + uint8 vmbits;
> OffsetNumber maxoff;
> OffsetNumber predecessor[MaxOffsetNumber];
> OffsetNumber successor[MaxOffsetNumber];
> @@ -499,6 +500,21 @@ verify_heapam(PG_FUNCTION_ARGS)
> ctx.blkno = BufferGetBlockNumber(ctx.buffer);
> ctx.page = BufferGetPage(ctx.buffer);
>
> + /*
> + * It is corruption of the page-level PD_ALL_VISIBLE flag is clear and
> + * the visibility map bits corresponding to this heap page are set.
> + */
> + vmbits = visibilitymap_get_status(ctx.rel, ctx.blkno, &vmbuffer);
Maybe it's worth adding a comment explaining that visibilitymap_get_status()
won't fail, because the VM code uses ZERO_ON_ERROR and
visibilitymap_get_status() accepts there no being underlying page?
That'd basically address why this isn't meaningfully harming this goal:
* This code goes to some trouble to avoid crashing the server even if the
* table pages are badly corrupted, but it's probably not perfect.
> + if (!PageIsAllVisible(ctx.page) &&
> + (vmbits & VISIBILITYMAP_ALL_VISIBLE) != 0)
> + {
> + ctx.offnum = InvalidOffsetNumber;
> + ctx.attnum = -1;
> + report_corruption(&ctx,
> + psprintf("page is not marked all-visible in page header but visibility map bit is set"));
I'd just check for both bits, not just all visible. Not that it's likely
you'd have just ALL_FROZEN set, but ...
> From 2be0dee6aa408752200ef9adfa9bf6a94716b4b2 Mon Sep 17 00:00:00 2001
> From: Melanie Plageman <melanieplageman@gmail.com>
> Date: Wed, 29 Apr 2026 10:33:46 -0400
> Subject: [PATCH v1_PGMASTER 6/7] Stop masking PD_ALL_VISIBLE when checking WAL
> consistency
>
> Setting and clearing PD_ALL_VISIBLE is WAL-logged, so there is no need
> to mask it for the purposes of WAL consistency checking.
>
> There is one minor exception, which is that when fixing VM corruption,
> we clear PD_ALL_VISIBLE on the heap page without specifically emitting
> WAL. This is a situation where corruption has already happened, however,
> so it is okay if WAL consistency checking flags this as well.
>
> Stop masking PD_ALL_VISIBLE, as doing so can only conceal real bugs.
>
> Author: Melanie Plageman <melanieplageman@gmail.com>
> ---
> src/backend/access/common/bufmask.c | 6 ------
> 1 file changed, 6 deletions(-)
>
> diff --git a/src/backend/access/common/bufmask.c b/src/backend/access/common/bufmask.c
> index 5f63d04c9cb..93e34ea38b8 100644
> --- a/src/backend/access/common/bufmask.c
> +++ b/src/backend/access/common/bufmask.c
> @@ -53,12 +53,6 @@ mask_page_hint_bits(Page page)
> /* Ignore PD_PAGE_FULL and PD_HAS_FREE_LINES flags, they are just hints. */
> PageClearFull(page);
> PageClearHasFreeLinePointers(page);
> -
> - /*
> - * PD_ALL_VISIBLE is masked during WAL consistency checking. XXX: It is
> - * worth investigating if we could stop doing this.
> - */
> - PageClearAllVisible(page);
I'm still baffled we did this. A real hear-no-evil-see-no-evil approach.
> diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
> index 8849610db00..63747aec566 100644
> --- a/src/backend/access/transam/xlogreader.c
> +++ b/src/backend/access/transam/xlogreader.c
> @@ -1794,6 +1794,10 @@ DecodeXLogRecord(XLogReaderState *state,
>
> blk = &decoded->blocks[block_id];
> blk->in_use = true;
> +#ifdef USE_ASSERT_CHECKING
> + blk->used_read = false;
> +#endif
> +
> blk->apply_image = false;
>
> COPY_HEADER_FIELD(&fork_flags, sizeof(uint8));
> diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
> index c236e2b7969..4e1d50f642d 100644
> --- a/src/backend/access/transam/xlogrecovery.c
> +++ b/src/backend/access/transam/xlogrecovery.c
> @@ -1962,6 +1962,15 @@ ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *repl
> /* Now apply the WAL record itself */
> GetRmgr(record->xl_rmid).rm_redo(xlogreader);
>
> +#ifdef USE_ASSERT_CHECKING
> + for (int block_id = 0; block_id <= xlogreader->record->max_block_id; block_id++)
> + {
> + DecodedBkpBlock *blk = &xlogreader->record->blocks[block_id];
> +
> + Assert(!blk->in_use || blk->used_read);
> + }
> +#endif
Maybe deserves a comment like:
Verify that all block references were used during replay. This helps detect
bugs in redo routines. Every referenced block needs to be replayed with
XLogReadBufferForRedoExtended(), possibly via a helper, to ensure that we
replay FPIs, extend the relation if necessary and other similar protections.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-05-06 21:46 Melanie Plageman <melanieplageman@gmail.com>
parent: Andres Freund <andres@anarazel.de>
0 siblings, 2 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-05-06 21:46 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; +Cc: Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
Thanks for the review! I incorporated all your suggestions in attached
v2 unless otherwise noted below.
Based on an LLM-assisted self-review of this code, I think there is a
pre-existing bug in heap_lock_updated_tuple_rec() where we don't reset
cleared_all_frozen and may have a stale value if we iterate more than
once (new tuple versions) or use the l4 goto label (retries for the
same tuple version). This could mean we unnecessarily clear all-frozen
on the standby (and unnecessarily register the VM buffer after my
patch). I couldn't come up with a repro though, so I didn't include a
patch to fix it, as I wanted to confirm that someone else also thought
this is a bug.
On Fri, May 1, 2026 at 3:14 PM Andres Freund <andres@anarazel.de> wrote:
>
> On 2026-04-30 17:44:28 -0400, Melanie Plageman wrote:
>
> > Attached is a patch set to fix the issue based largely on the work you
> > started on your branch. I attached the version targeting master/19
> > which is prefixed with v1_PGMASTER and the version targeting 18,
> > prefixed v1_PG18. The pg 18 changes aren't a straight cherry-pick to
> > 17 (the earliest I'll backpatch because that was when incremental
> > backup was introduced) because the redo functions live in a different
> > file in 18 than in 17, but I want to avoid discussing three different
> > versions of this patch set on this thread.
>
> Makes sense.
>
> I think it's worth explicitly bringing up that this means that we leave an
> arguable corruption case in the earlier branches.
But can there be such a case since we read the VM with ZERO_AND_LOCK?
Are there other cases that rely only on the buffers registered? Is it
possible it isn't reachable before 17?
> I thought I'd start looking at the 18 patches.
Attached v2_PG18 and v2_PG19 include all the updates from your
previous review applied to both branches.
> > @@ -331,7 +332,10 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
> >
> > UnlockReleaseBuffer(buf);
> >
> > - if (vmbuf != InvalidBuffer)
> > +
> > + if (did_modify_vm)
> > + UnlockReleaseBuffer(vmbuf);
> > + else if (BufferIsValid(vmbuf))
> > ReleaseBuffer(vmbuf);
> >
> > /* Update the current_start_ptr before moving to the next page. */
>
> Why did you touch heap_surgery in 18? That shouldn't be necessary, right?
It does visibilitymap_clear() and is also broken. Though, as you
pointed out off-list, my fix wasn't even far enough. It logs a WAL
record for the heap page and a separate one for the VM update which
isn't correct. v2 has a single WAL record with both registered.
> It probably doesn't matter, but it seems like it'd be ever so slightly more
> correct to first do visibilitymap_clear_locked() and then
> PageClearAllVisible(), given
> a) the rule that there should never be a VM bit set when !PD_ALL_VISIBLE
> b) VM reads happening without page locks
>
> I don't think it can ever lead to a wrong result, but still seems better to do
> it the "right" way round.
I've changed this throughout the patch set.
> > @@ -2228,13 +2231,22 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
> > /* filtering by origin on a row level is much more efficient */
> > XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
> >
> > + if (all_visible_cleared)
> > + XLogRegisterBuffer(1, vmbuffer, 0);
> > +
>
> For update you created HEAP_UPDATE_BLKREF_* to distinguish the different block
> refernces. I think that was a good call to make the code at least somewhat
> understandable. I think once we do it for heap_update(), we should just do it
> for heap_{insert,multi_insert,delete,update} too.
I've done this in both versions as 0001.
> Am I blind, or have not documented the lock ordering rules for heap vs VM
> anywhere?
I don't see anywhere where something like this is documented. Though
I'm not entirely sure what you mean. In the cases with one VM block
and one heap block, you mean we should have documented that you should
acquire both locks before making changes to either page?
Thinking about this made me wonder if I should refactor the patches a
bit to acquire the vmbuffer lock before the critical section instead.
This would make the heap_insert() go from this
START_CRIT_SECTION();
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
if (PageIsAllVisible(page))
{
all_visible_cleared = true;
LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
visibilitymap_clear(relation->rd_locator,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS,
InvalidXLogRecPtr);
PageClearAllVisible(BufferGetPage(buffer));
}
to something more like this:
if (PageIsAllVisible(page))
{
clear_all_visible = true;
LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
}
START_CRIT_SECTION();
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
if (clear_all_visible)
{
visibilitymap_clear(relation->rd_locator,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS,
InvalidXLogRecPtr);
PageClearAllVisible(BufferGetPage(buffer));
}
> I think here we actually have a lock ordering issue:
<-- snip -->
> So I think we need to be careful and acquire the exlusive locks on the two vm
> pages in ascending block number order. That'd turn the above into:
>
> B1: lock VM1
> B2: lock VM1 <waits>
> B1: lock VM2
> B1: xloginsert etc
> B1: release locks
> B2: lock VM1 <completes>
> B2: lock VM2
Right, good catch. I'd not thought of that at all. I've done this in
attached v2.
> > @@ -363,9 +363,24 @@ heap_xlog_delete(XLogReaderState *record)
> > Relation reln = CreateFakeRelcacheEntry(target_locator);
> > Buffer vmbuffer = InvalidBuffer;
> >
> > - visibilitymap_pin(reln, blkno, &vmbuffer);
> > - visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
> > - ReleaseBuffer(vmbuffer);
> > + if (XLogRecHasBlockRef(record, 1))
> I think we need a comment explaining why we need this if
> (XLogRecHasBlockRef()) dance. Otherwise whoever has to look at this for some
> bug in the future will be rather confused.
I've added a comment by the new helper function that I added.
> Perhaps just have one comment, with the other places referring to it? Might
> also be sane to leave a breadcrumb on the WAL logging side, that it used to
> look different in the same major version.
What do you mean by the WAL logging side? This already is the redo
routine. Or do you mean something else by breadcrumb?
> Wonder if we could package this into some kind of helper, there are a bunch of
> copies of this now. But there's enough small differences that it's not
> trivial.
I did this.
Just to confirm, we do _not_ want to touch XLOG_PAGE_MAGIC on the back
branches because it specifically doesn't change the WAL format
compatibility.
> Hm. Could this be simplified to something like:
>
> if (has_vm_new)
> {
> Buffer vmbuffer = InvalidBuffer;
>
> if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW, &vmbuffer) ==
> BLK_NEEDS_REDO)
> {
> if (new_cleared)
> visibilitymap_clear_locked(reln, newblk, vmbuffer,
> VISIBILITYMAP_VALID_BITS);
> if (old_cleared && !has_vm_old)
> visibilitymap_clear_locked(reln, oldblk, vmbuffer,
> VISIBILITYMAP_VALID_BITS);
> PageSetLSN(BufferGetPage(vmbuffer), lsn);
> }
>
> if (BufferIsValid(vmbuffer))
> UnlockReleaseBuffer(vmbuffer);
> }
>
> if (has_vm_old)
> {
> Buffer vmbuffer = InvalidBuffer;
>
> Assert(old_cleared);
>
> if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_OLD, &vmbuffer) ==
> BLK_NEEDS_REDO)
> {
> visibilitymap_clear_locked(reln, oldblk, vmbuffer,
> VISIBILITYMAP_VALID_BITS);
> PageSetLSN(BufferGetPage(vmbuffer), lsn);
> }
>
> if (BufferIsValid(vmbuffer))
> UnlockReleaseBuffer(vmbuffer);
> }
>
> if (!has_vm_new && !has_vm_old)
> {
I tried this ordering. Personally, I find my current way more clear.
Then the different cases are self-contained. If both the vmbits for
two heap pages are set on the same VM page, you can understand what is
registered without looking across multiple if statements.
I also prefer to have the same structure in replay as in normal
operation. And in normal operation, because of the page lock ordering
concerns, the single VM page vs two VM page cases are even more
different.
> Would it make sense to have an assert like
>
> Assert(has_vm_new || !has_vm_old); /* should never just have old VM buffer */
It is okay to have just VM old if only the VM bits on the old page are
set (because there are two VM buffers but the new one is not
all-visible).
> > +bool
> > +visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
>
> Given this is a new API, I wonder if we shouldn't already start using the new
> signature. If there's any extensions using it (I think there is at least one),
> this would just require them to have an ifdef for <17, 17-18, and >= 19.
So in 19, I just changed the function signature of
visibilitymap_clear() a bit and did not add a
visibilitymap_clear_locked(). This was doable because I felt more
comfortable changing 19 to not take a Relation and instead take a
RelFileLocator as a parameter because visibilitymap_set() already does
this on 19.
I am wondering in 19 if we should just set the page LSN in the call
sites and not try to have visibilitymap_clear() take an LSN. You had a
separate visibilitymap_clear_redo() function that took an LSN. I guess
I could make a separate function that just calls visibilitymap_clear()
and then does PageSetLSN(), but that seemed kind of silly. What do you
think?
Separately, in 18, I say visibilitymap_clear() is there for backwards
compatibility but then I use it in vacuumlazy.c where we clear the VM
without WAL logging when it is corrupted. Should I not use it there
because it is supposed ot be only for backwards compatibility? Or does
it not matter...
> > +my ($post_del_vis, $post_del_frz) = get_vm_summary($primary, 'vm_delete_test');
> > +cmp_ok($post_del_vis, '<', $pre_del_vis,
> > + "DELETE test: all-visible cleared after delete");
> > +cmp_ok($post_del_frz, '<', $pre_del_frz,
> > + "DELETE test: all-frozen cleared after delete");
> > +
> > +my ($incr2, $incr2_name) = take_incr_backup($full2);
> > +validate_restored_vm($full2, $full2_name, $incr2, $incr2_name,
> > + 'vm_delete_test', 'DELETE test');
>
> Hm. Since these are looking at different tables, can't we do that with one
> backup + incremental backup + combine + validate cycle for all of them?
I've done this
> Also seems like the modifications, tests and messages for all of them could be
> moved to a helper?
Also did this.
> I assume you verified that the tests catch the problem without the fix?
Yes, though the update case was doing lock first and thus not
exercising exactly what I intended. I think it is fixed now.
The Cluster.pm changes that pipes the pg_combinebackup output to a
temp file and then prints that to the regress log if a test fails
still need review. I had some AI-assistance writing it, and I'm pretty
sure it behaves as I intended but I don't know if it is idiomatic perl
or how we do things or whatever.
> > @@ -499,6 +500,21 @@ verify_heapam(PG_FUNCTION_ARGS)
> > ctx.blkno = BufferGetBlockNumber(ctx.buffer);
> > ctx.page = BufferGetPage(ctx.buffer);
> >
> > + /*
> > + * It is corruption of the page-level PD_ALL_VISIBLE flag is clear and
> > + * the visibility map bits corresponding to this heap page are set.
> > + */
> > + vmbits = visibilitymap_get_status(ctx.rel, ctx.blkno, &vmbuffer);
>
> Maybe it's worth adding a comment explaining that visibilitymap_get_status()
> won't fail, because the VM code uses ZERO_ON_ERROR and
> visibilitymap_get_status() accepts there no being underlying page?
>
> That'd basically address why this isn't meaningfully harming this goal:
> * This code goes to some trouble to avoid crashing the server even if the
> * table pages are badly corrupted, but it's probably not perfect.
Well, I wouldn't necessarily expect visibilitymap_get_status() to fail
because of PD_ALL_VISIBLE being clear on the heap pages. And
ZERO_ON_ERROR isn't why that doesn't happen. But I think your point is
we should explain to the reader that visibilitymap_get_status() itself
won't assert/ERROR out when called on either a corrupted heap page or
corrupted VM page. Or is that not what you are saying?
- Melanie
Attachments: download all (12 files, .tar.gz)
v2_PG19-0001-Alias-WAL-block-references-for-some-record-t.patch (text/x-patch, 15.3K) download
From 8e1591d02ee3c1761bc982445438cb6cfb7d1442 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Mon, 4 May 2026 10:41:13 -0400
Subject: [PATCH v2_PG19 1/8] Alias WAL block references for some record types
A future commit will add block references for pages of the VM to update,
insert, delete, lock, and multi-insert WAL records. Before doing that,
alias the heap block references so that they are easy to distinguish.
Author: Melanie Plageman <melanieplageman@gmail.com>
Discussion: https://postgr.es/m/66mqpfyti3qhfttcsv6r2lbvqqd32rrmpn6i47ovrsnvguts46%40gou54xc5jld3
Backpatch through: 17
---
src/backend/access/heap/heapam.c | 45 ++++++++++++---------
src/backend/access/heap/heapam_xlog.c | 56 +++++++++++++++++----------
src/include/access/heapam_xlog.h | 32 +++++++++++----
3 files changed, 87 insertions(+), 46 deletions(-)
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index abfd8e8970a..30cc651f078 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2149,10 +2149,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
* write the whole page to the xlog, we don't need to store
* xl_heap_header in the xlog.
*/
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
- XLogRegisterBufData(0, &xlhdr, SizeOfHeapHeader);
+ XLogRegisterBuffer(HEAP_INSERT_BLKREF_HEAP, buffer,
+ REGBUF_STANDARD | bufflags);
+ XLogRegisterBufData(HEAP_INSERT_BLKREF_HEAP, &xlhdr,
+ SizeOfHeapHeader);
/* PG73FORMAT: write bitmap [+ padding] [+ oid] + data */
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_INSERT_BLKREF_HEAP,
(char *) heaptup->t_data + SizeofHeapTupleHeader,
heaptup->t_len - SizeofHeapTupleHeader);
@@ -2573,11 +2575,13 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogBeginInsert();
XLogRegisterData(xlrec, tupledata - scratch.data);
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
+ XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_HEAP, buffer,
+ REGBUF_STANDARD | bufflags);
if (all_frozen_set)
- XLogRegisterBuffer(1, vmbuffer, 0);
+ XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_VM, vmbuffer, 0);
- XLogRegisterBufData(0, tupledata, totaldatalen);
+ XLogRegisterBufData(HEAP_MULTI_INSERT_BLKREF_HEAP, tupledata,
+ totaldatalen);
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
@@ -3069,7 +3073,7 @@ l1:
XLogBeginInsert();
XLogRegisterData(&xlrec, SizeOfHeapDelete);
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_DELETE_BLKREF_HEAP, buffer, REGBUF_STANDARD);
/*
* Log replica identity of the deleted tuple if there is one
@@ -3844,7 +3848,7 @@ l2:
XLogRecPtr recptr;
XLogBeginInsert();
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buffer, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&oldtup.t_self);
xlrec.xmax = xmax_lock_old_tuple;
@@ -5183,7 +5187,7 @@ failed:
XLogRecPtr recptr;
XLogBeginInsert();
- XLogRegisterBuffer(0, *buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, *buffer, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&tuple->t_self);
xlrec.xmax = xid;
@@ -5935,7 +5939,7 @@ l4:
Page page = BufferGetPage(buf);
XLogBeginInsert();
- XLogRegisterBuffer(0, buf, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buf, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&mytup.t_self);
xlrec.xmax = new_xmax;
@@ -8901,9 +8905,9 @@ log_heap_update(Relation reln, Buffer oldbuf,
if (need_tuple_data)
bufflags |= REGBUF_KEEP_DATA;
- XLogRegisterBuffer(0, newbuf, bufflags);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_NEW, newbuf, bufflags);
if (oldbuf != newbuf)
- XLogRegisterBuffer(1, oldbuf, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_OLD, oldbuf, REGBUF_STANDARD);
XLogRegisterData(&xlrec, SizeOfHeapUpdate);
@@ -8916,15 +8920,18 @@ log_heap_update(Relation reln, Buffer oldbuf,
{
prefix_suffix[0] = prefixlen;
prefix_suffix[1] = suffixlen;
- XLogRegisterBufData(0, &prefix_suffix, sizeof(uint16) * 2);
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW, &prefix_suffix,
+ sizeof(uint16) * 2);
}
else if (prefixlen > 0)
{
- XLogRegisterBufData(0, &prefixlen, sizeof(uint16));
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW, &prefixlen,
+ sizeof(uint16));
}
else
{
- XLogRegisterBufData(0, &suffixlen, sizeof(uint16));
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW, &suffixlen,
+ sizeof(uint16));
}
}
@@ -8938,10 +8945,10 @@ log_heap_update(Relation reln, Buffer oldbuf,
*
* The 'data' doesn't include the common prefix or suffix.
*/
- XLogRegisterBufData(0, &xlhdr, SizeOfHeapHeader);
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW, &xlhdr, SizeOfHeapHeader);
if (prefixlen == 0)
{
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW,
(char *) newtup->t_data + SizeofHeapTupleHeader,
newtup->t_len - SizeofHeapTupleHeader - suffixlen);
}
@@ -8954,13 +8961,13 @@ log_heap_update(Relation reln, Buffer oldbuf,
/* bitmap [+ padding] [+ oid] */
if (newtup->t_data->t_hoff - SizeofHeapTupleHeader > 0)
{
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW,
(char *) newtup->t_data + SizeofHeapTupleHeader,
newtup->t_data->t_hoff - SizeofHeapTupleHeader);
}
/* data after common prefix */
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW,
(char *) newtup->t_data + newtup->t_data->t_hoff + prefixlen,
newtup->t_len - newtup->t_data->t_hoff - prefixlen - suffixlen);
}
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index 9ed7024e814..dd027dd5f50 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -299,7 +299,8 @@ heap_xlog_delete(XLogReaderState *record)
RelFileLocator target_locator;
ItemPointerData target_tid;
- XLogRecGetBlockTag(record, 0, &target_locator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_DELETE_BLKREF_HEAP, &target_locator, NULL,
+ &blkno);
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
@@ -318,7 +319,8 @@ heap_xlog_delete(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_DELETE_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = BufferGetPage(buffer);
@@ -383,7 +385,8 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerData target_tid;
XLogRedoAction action;
- XLogRecGetBlockTag(record, 0, &target_locator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_INSERT_BLKREF_HEAP, &target_locator, NULL,
+ &blkno);
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
@@ -411,13 +414,14 @@ heap_xlog_insert(XLogReaderState *record)
*/
if (XLogRecGetInfo(record) & XLOG_HEAP_INIT_PAGE)
{
- buffer = XLogInitBufferForRedo(record, 0);
+ buffer = XLogInitBufferForRedo(record, HEAP_INSERT_BLKREF_HEAP);
page = BufferGetPage(buffer);
PageInit(page, BufferGetPageSize(buffer), 0);
action = BLK_NEEDS_REDO;
}
else
- action = XLogReadBufferForRedo(record, 0, &buffer);
+ action = XLogReadBufferForRedo(record, HEAP_INSERT_BLKREF_HEAP,
+ &buffer);
if (action == BLK_NEEDS_REDO)
{
Size datalen;
@@ -428,7 +432,7 @@ heap_xlog_insert(XLogReaderState *record)
if (PageGetMaxOffsetNumber(page) + 1 < xlrec->offnum)
elog(PANIC, "invalid max offset number");
- data = XLogRecGetBlockData(record, 0, &datalen);
+ data = XLogRecGetBlockData(record, HEAP_INSERT_BLKREF_HEAP, &datalen);
newlen = datalen - SizeOfHeapHeader;
Assert(datalen > SizeOfHeapHeader && newlen <= MaxHeapTupleSize);
@@ -516,7 +520,8 @@ heap_xlog_multi_insert(XLogReaderState *record)
*/
xlrec = (xl_heap_multi_insert *) XLogRecGetData(record);
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_MULTI_INSERT_BLKREF_HEAP, &rlocator, NULL,
+ &blkno);
/* check that the mutually exclusive flags are not both set */
Assert(!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
@@ -539,13 +544,14 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (isinit)
{
- buffer = XLogInitBufferForRedo(record, 0);
+ buffer = XLogInitBufferForRedo(record, HEAP_MULTI_INSERT_BLKREF_HEAP);
page = BufferGetPage(buffer);
PageInit(page, BufferGetPageSize(buffer), 0);
action = BLK_NEEDS_REDO;
}
else
- action = XLogReadBufferForRedo(record, 0, &buffer);
+ action = XLogReadBufferForRedo(record, HEAP_MULTI_INSERT_BLKREF_HEAP,
+ &buffer);
if (action == BLK_NEEDS_REDO)
{
char *tupdata;
@@ -553,7 +559,8 @@ heap_xlog_multi_insert(XLogReaderState *record)
Size len;
/* Tuples are stored as block data */
- tupdata = XLogRecGetBlockData(record, 0, &len);
+ tupdata = XLogRecGetBlockData(record, HEAP_MULTI_INSERT_BLKREF_HEAP,
+ &len);
endptr = tupdata + len;
page = BufferGetPage(buffer);
@@ -728,8 +735,10 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
oldtup.t_data = NULL;
oldtup.t_len = 0;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &newblk);
- if (XLogRecGetBlockTagExtended(record, 1, NULL, NULL, &oldblk, NULL))
+ XLogRecGetBlockTag(record, HEAP_UPDATE_BLKREF_NEW, &rlocator, NULL,
+ &newblk);
+ if (XLogRecGetBlockTagExtended(record, HEAP_UPDATE_BLKREF_OLD, NULL, NULL,
+ &oldblk, NULL))
{
/* HOT updates are never done across pages */
Assert(!hot_update);
@@ -765,7 +774,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
*/
/* Deal with old tuple version */
- oldaction = XLogReadBufferForRedo(record, (oldblk == newblk) ? 0 : 1,
+ oldaction = XLogReadBufferForRedo(record, (oldblk == newblk) ?
+ HEAP_UPDATE_BLKREF_NEW : HEAP_UPDATE_BLKREF_OLD,
&obuffer);
if (oldaction == BLK_NEEDS_REDO)
{
@@ -815,13 +825,14 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
}
else if (XLogRecGetInfo(record) & XLOG_HEAP_INIT_PAGE)
{
- nbuffer = XLogInitBufferForRedo(record, 0);
+ nbuffer = XLogInitBufferForRedo(record, HEAP_UPDATE_BLKREF_NEW);
npage = BufferGetPage(nbuffer);
PageInit(npage, BufferGetPageSize(nbuffer), 0);
newaction = BLK_NEEDS_REDO;
}
else
- newaction = XLogReadBufferForRedo(record, 0, &nbuffer);
+ newaction = XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_NEW,
+ &nbuffer);
/*
* The visibility map may need to be fixed even if the heap page is
@@ -846,7 +857,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Size datalen;
Size tuplen;
- recdata = XLogRecGetBlockData(record, 0, &datalen);
+ recdata = XLogRecGetBlockData(record, HEAP_UPDATE_BLKREF_NEW,
+ &datalen);
recdata_end = recdata + datalen;
npage = BufferGetPage(nbuffer);
@@ -1033,7 +1045,8 @@ heap_xlog_lock(XLogReaderState *record)
BlockNumber block;
Relation reln;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
+ XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
+ &block);
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
@@ -1043,7 +1056,8 @@ heap_xlog_lock(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = BufferGetPage(buffer);
@@ -1109,7 +1123,8 @@ heap_xlog_lock_updated(XLogReaderState *record)
BlockNumber block;
Relation reln;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
+ XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
+ &block);
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
@@ -1119,7 +1134,8 @@ heap_xlog_lock_updated(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = BufferGetPage(buffer);
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index fdca7d821c8..d425562fd79 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -112,6 +112,9 @@
(XLH_DELETE_CONTAINS_OLD_TUPLE | XLH_DELETE_CONTAINS_OLD_KEY)
/* This is what we need to know about delete */
+#define HEAP_DELETE_BLKREF_HEAP 0
+#define HEAP_DELETE_BLKREF_VM 1
+
typedef struct xl_heap_delete
{
TransactionId xmax; /* xmax of the deleted tuple */
@@ -159,12 +162,15 @@ typedef struct xl_heap_header
#define SizeOfHeapHeader (offsetof(xl_heap_header, t_hoff) + sizeof(uint8))
/* This is what we need to know about insert */
+#define HEAP_INSERT_BLKREF_HEAP 0
+#define HEAP_INSERT_BLKREF_VM 1
+
typedef struct xl_heap_insert
{
OffsetNumber offnum; /* inserted tuple's offset */
uint8 flags;
- /* xl_heap_header & TUPLE DATA in backup block 0 */
+ /* xl_heap_header & TUPLE DATA in HEAP_INSERT_BLKREF_HEAP */
} xl_heap_insert;
#define SizeOfHeapInsert (offsetof(xl_heap_insert, flags) + sizeof(uint8))
@@ -175,10 +181,14 @@ typedef struct xl_heap_insert
* The main data of the record consists of this xl_heap_multi_insert header.
* 'offsets' array is omitted if the whole page is reinitialized
* (XLOG_HEAP_INIT_PAGE).
- *
- * In block 0's data portion, there is an xl_multi_insert_tuple struct,
- * followed by the tuple data for each tuple. There is padding to align
- * each xl_multi_insert_tuple struct.
+ */
+#define HEAP_MULTI_INSERT_BLKREF_HEAP 0
+#define HEAP_MULTI_INSERT_BLKREF_VM 1
+
+/*
+ * In HEAP_MULTI_INSERT_BLKREF_HEAP's data portion, there is an
+ * xl_multi_insert_tuple struct, followed by the tuple data for each tuple.
+ * There is padding to align each xl_multi_insert_tuple struct.
*/
typedef struct xl_heap_multi_insert
{
@@ -203,7 +213,7 @@ typedef struct xl_multi_insert_tuple
/*
* This is what we need to know about update|hot_update
*
- * Backup blk 0: new page
+ * HEAP_UPDATE_BLKREF_NEW: new page
*
* If XLH_UPDATE_PREFIX_FROM_OLD or XLH_UPDATE_SUFFIX_FROM_OLD flags are set,
* the prefix and/or suffix come first, as one or two uint16s.
@@ -215,8 +225,13 @@ typedef struct xl_multi_insert_tuple
* If XLH_UPDATE_CONTAINS_NEW_TUPLE flag is given, the tuple data is
* included even if a full-page image was taken.
*
- * Backup blk 1: old page, if different. (no data, just a reference to the blk)
+ * HEAP_UPDATE_BLKREF_OLD: old page, if different. (no data, just a reference
+ * to the block)
*/
+
+#define HEAP_UPDATE_BLKREF_NEW 0
+#define HEAP_UPDATE_BLKREF_OLD 1
+
typedef struct xl_heap_update
{
TransactionId old_xmax; /* xmax of the old tuple */
@@ -403,6 +418,9 @@ typedef struct xlhp_prune_items
#define XLH_LOCK_ALL_FROZEN_CLEARED 0x01
/* This is what we need to know about lock */
+#define HEAP_LOCK_BLKREF_HEAP 0
+#define HEAP_LOCK_BLKREF_VM 1
+
typedef struct xl_heap_lock
{
TransactionId xmax; /* might be a MultiXactId */
--
2.43.0
v2_PG19-0002-Make-VM-clear-take-a-RelFileLocator-and-not-.patch (text/x-patch, 10.1K) download
From 42603d4466014baa995f88823c204168c3f49083 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Mon, 27 Apr 2026 17:30:17 -0400
Subject: [PATCH v2_PG19 2/8] Make VM clear take a RelFileLocator and not fake
relcache
This brings it in line with visibilitymap_set(). An upcoming commit will
start reading visibility map buffers with XLogReadBuffer*() functions,
so we no longer require a relation object to pin the visibility map,
allowing us to remove this fake relcache business altogether.
While we're at it, remove a spurious const qualifier from the
RelFileLocator parameter to visibilitymap_set().
Author: Melanie Plageman <melanieplageman@gmail.com>
Discussion: https://postgr.es/m/flat/CAAKRu_bn%2Be7F4yPFBgFbnP%2BsyJRKyNK092bjD2LKvZW7O4Svag%40mail.gmail.com
---
contrib/pg_surgery/heap_surgery.c | 2 +-
src/backend/access/heap/heapam.c | 16 ++++++++--------
src/backend/access/heap/heapam_xlog.c | 14 +++++++-------
src/backend/access/heap/pruneheap.c | 2 +-
src/backend/access/heap/visibilitymap.c | 9 ++++++---
src/include/access/visibilitymap.h | 4 ++--
6 files changed, 25 insertions(+), 22 deletions(-)
diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c
index ae4e7c0136c..b82c29e7cbf 100644
--- a/contrib/pg_surgery/heap_surgery.c
+++ b/contrib/pg_surgery/heap_surgery.c
@@ -267,7 +267,7 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
if (PageIsAllVisible(page))
{
PageClearAllVisible(page);
- visibilitymap_clear(rel, blkno, vmbuf,
+ visibilitymap_clear(rel->rd_locator, blkno, vmbuf,
VISIBILITYMAP_VALID_BITS);
did_modify_vm = true;
}
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 30cc651f078..275bc68345a 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2063,7 +2063,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
{
all_visible_cleared = true;
PageClearAllVisible(page);
- visibilitymap_clear(relation,
+ visibilitymap_clear(relation->rd_locator,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
@@ -2442,7 +2442,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
{
all_visible_cleared = true;
PageClearAllVisible(page);
- visibilitymap_clear(relation,
+ visibilitymap_clear(relation->rd_locator,
BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
@@ -3002,7 +3002,7 @@ l1:
{
all_visible_cleared = true;
PageClearAllVisible(page);
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
+ visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
@@ -3836,7 +3836,7 @@ l2:
* worthwhile.
*/
if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
+ visibilitymap_clear(relation->rd_locator, block, vmbuffer,
VISIBILITYMAP_ALL_FROZEN))
cleared_all_frozen = true;
@@ -4068,14 +4068,14 @@ l2:
{
all_visible_cleared = true;
PageClearAllVisible(page);
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
+ visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(buffer),
vmbuffer, VISIBILITYMAP_VALID_BITS);
}
if (newbuf != buffer && PageIsAllVisible(newpage))
{
all_visible_cleared_new = true;
PageClearAllVisible(newpage);
- visibilitymap_clear(relation, BufferGetBlockNumber(newbuf),
+ visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(newbuf),
vmbuffer_new, VISIBILITYMAP_VALID_BITS);
}
@@ -5162,7 +5162,7 @@ failed:
/* Clear only the all-frozen bit on visibility map if needed */
if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
+ visibilitymap_clear(relation->rd_locator, block, vmbuffer,
VISIBILITYMAP_ALL_FROZEN))
cleared_all_frozen = true;
@@ -5916,7 +5916,7 @@ l4:
&new_xmax, &new_infomask, &new_infomask2);
if (PageIsAllVisible(BufferGetPage(buf)) &&
- visibilitymap_clear(rel, block, vmbuffer,
+ visibilitymap_clear(rel->rd_locator, block, vmbuffer,
VISIBILITYMAP_ALL_FROZEN))
cleared_all_frozen = true;
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index dd027dd5f50..4e994cd01f3 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -314,7 +314,7 @@ heap_xlog_delete(XLogReaderState *record)
Buffer vmbuffer = InvalidBuffer;
visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear(target_locator, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -403,7 +403,7 @@ heap_xlog_insert(XLogReaderState *record)
Buffer vmbuffer = InvalidBuffer;
visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear(target_locator, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -536,7 +536,7 @@ heap_xlog_multi_insert(XLogReaderState *record)
Relation reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear(rlocator, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
ReleaseBuffer(vmbuffer);
vmbuffer = InvalidBuffer;
FreeFakeRelcacheEntry(reln);
@@ -758,7 +758,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Buffer vmbuffer = InvalidBuffer;
visibilitymap_pin(reln, oldblk, &vmbuffer);
- visibilitymap_clear(reln, oldblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear(rlocator, oldblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -844,7 +844,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Buffer vmbuffer = InvalidBuffer;
visibilitymap_pin(reln, newblk, &vmbuffer);
- visibilitymap_clear(reln, newblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear(rlocator, newblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -1050,7 +1050,7 @@ heap_xlog_lock(XLogReaderState *record)
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
+ visibilitymap_clear(rlocator, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
@@ -1128,7 +1128,7 @@ heap_xlog_lock_updated(XLogReaderState *record)
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
+ visibilitymap_clear(rlocator, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c
index fdddd23035b..184fb70a24f 100644
--- a/src/backend/access/heap/pruneheap.c
+++ b/src/backend/access/heap/pruneheap.c
@@ -927,7 +927,7 @@ heap_page_fix_vm_corruption(PruneState *prstate, OffsetNumber offnum,
if (do_clear_vm)
{
- visibilitymap_clear(prstate->relation, prstate->block,
+ visibilitymap_clear(prstate->relation->rd_locator, prstate->block,
prstate->vmbuffer,
VISIBILITYMAP_VALID_BITS);
prstate->old_vmbits = 0;
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 4fd470702aa..3b699291b23 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -148,7 +148,8 @@ static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks);
* any I/O. Returns true if any bits have been cleared and false otherwise.
*/
bool
-visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
+visibilitymap_clear(RelFileLocator rlocator,
+ BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
{
BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
@@ -162,7 +163,9 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
Assert(flags != VISIBILITYMAP_ALL_VISIBLE);
#ifdef TRACE_VISIBILITYMAP
- elog(DEBUG1, "vm_clear %s %d", RelationGetRelationName(rel), heapBlk);
+ elog(DEBUG1, "vm_clear %s %d",
+ relpathbackend(rlocator, MyProcNumber, MAIN_FORKNUM).str,
+ heapBlk);
#endif
if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
@@ -254,7 +257,7 @@ visibilitymap_pin_ok(BlockNumber heapBlk, Buffer vmbuf)
void
visibilitymap_set(BlockNumber heapBlk,
Buffer vmBuf, uint8 flags,
- const RelFileLocator rlocator)
+ RelFileLocator rlocator)
{
BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
uint32 mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index e4e0cfa989e..b860c4ef3ca 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -26,14 +26,14 @@
#define VM_ALL_FROZEN(r, b, v) \
((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0)
-extern bool visibilitymap_clear(Relation rel, BlockNumber heapBlk,
+extern bool visibilitymap_clear(RelFileLocator rlocator, BlockNumber heapBlk,
Buffer vmbuf, uint8 flags);
extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk,
Buffer *vmbuf);
extern bool visibilitymap_pin_ok(BlockNumber heapBlk, Buffer vmbuf);
extern void visibilitymap_set(BlockNumber heapBlk,
Buffer vmBuf, uint8 flags,
- const RelFileLocator rlocator);
+ RelFileLocator rlocator);
extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf);
extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen);
extern BlockNumber visibilitymap_prepare_truncate(Relation rel,
--
2.43.0
v2_PG19-0003-Fix-WAL-logging-of-VM-clears.patch (text/x-patch, 33.4K) download
From 429a26229f798e4f0a7f90f8d97ad82644e67fae Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 12:53:49 -0400
Subject: [PATCH v2_PG19 3/8] Fix WAL logging of VM clears
Previously, we failed to register the visibility map buffer when
emitting WAL after clearing the VM bits for heap pages (recovery code
read the VM page normally). This meant that we couldn't emit FPIs of VM
pages when clearing VM bits. While this would not result in a checksum
error because the visibility map is read with ZERO_ON_ERROR, the WAL
summarizer only includes pages which were registered, so a restore from
an incremental backup would not include the modified VM pages -- leading
to data corruption.
Fix this by registering the VM buffer in the WAL record when clearing VM
bits. This also means the VM buffer must be locked throughout the
duration of the critical section where we make changes to the VM and
heap page and emit WAL.
Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/flat/CAAKRu_bn%2Be7F4yPFBgFbnP%2BsyJRKyNK092bjD2LKvZW7O4Svag%40mail.gmail.com
Backpatch-through: 17
---
contrib/pg_surgery/heap_surgery.c | 34 +++-
src/backend/access/heap/heapam.c | 256 ++++++++++++++++++++----
src/backend/access/heap/heapam_xlog.c | 151 +++++++++-----
src/backend/access/heap/pruneheap.c | 2 +-
src/backend/access/heap/visibilitymap.c | 36 ++--
src/bin/pg_walsummary/t/002_blocks.pl | 7 +-
src/include/access/heapam_xlog.h | 15 +-
src/include/access/visibilitymap.h | 3 +-
src/include/access/xlog_internal.h | 2 +-
9 files changed, 381 insertions(+), 125 deletions(-)
diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c
index b82c29e7cbf..4a8aa888213 100644
--- a/contrib/pg_surgery/heap_surgery.c
+++ b/contrib/pg_surgery/heap_surgery.c
@@ -17,6 +17,7 @@
#include "access/visibilitymap.h"
#include "access/xloginsert.h"
#include "catalog/pg_am_d.h"
+#include "catalog/pg_control.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
#include "utils/acl.h"
@@ -233,11 +234,14 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
}
/*
- * Before entering the critical section, pin the visibility map page
- * if it appears to be necessary.
+ * Before entering the critical section, pin and lock the visibility
+ * map page if it appears to be necessary.
*/
if (heap_force_opt == HEAP_FORCE_KILL && PageIsAllVisible(page))
+ {
visibilitymap_pin(rel, blkno, &vmbuf);
+ LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+ }
/* No ereport(ERROR) from here until all the changes are logged. */
START_CRIT_SECTION();
@@ -266,10 +270,11 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
*/
if (PageIsAllVisible(page))
{
- PageClearAllVisible(page);
visibilitymap_clear(rel->rd_locator, blkno, vmbuf,
- VISIBILITYMAP_VALID_BITS);
+ VISIBILITYMAP_VALID_BITS,
+ InvalidXLogRecPtr);
did_modify_vm = true;
+ PageClearAllVisible(page);
}
}
else
@@ -320,19 +325,26 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
/* XLOG stuff */
if (RelationNeedsWAL(rel))
- log_newpage_buffer(buf, true);
+ {
+ XLogRecPtr recptr;
+
+ XLogBeginInsert();
+ XLogRegisterBuffer(0, buf, REGBUF_STANDARD | REGBUF_FORCE_IMAGE);
+ if (did_modify_vm)
+ XLogRegisterBuffer(1, vmbuf, REGBUF_FORCE_IMAGE);
+ recptr = XLogInsert(RM_XLOG_ID, XLOG_FPI);
+ if (did_modify_vm)
+ PageSetLSN(BufferGetPage(vmbuf), recptr);
+ PageSetLSN(BufferGetPage(buf), recptr);
+ }
}
- /* WAL log the VM page if it was modified. */
- if (did_modify_vm && RelationNeedsWAL(rel))
- log_newpage_buffer(vmbuf, false);
-
END_CRIT_SECTION();
UnlockReleaseBuffer(buf);
- if (vmbuf != InvalidBuffer)
- ReleaseBuffer(vmbuf);
+ if (BufferIsValid(vmbuf))
+ UnlockReleaseBuffer(vmbuf);
/* Update the current_start_ptr before moving to the next page. */
curr_start_ptr = next_start_ptr;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 275bc68345a..48d619b00ab 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -60,7 +60,8 @@
static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup,
TransactionId xid, CommandId cid, uint32 options);
static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup,
+ Buffer vmbuffer_old, Buffer newbuf,
+ Buffer vmbuffer_new, HeapTuple oldtup,
HeapTuple newtup, HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared,
bool walLogical);
@@ -2062,10 +2063,13 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
if (PageIsAllVisible(page))
{
all_visible_cleared = true;
- PageClearAllVisible(page);
+
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
visibilitymap_clear(relation->rd_locator,
ItemPointerGetBlockNumber(&(heaptup->t_self)),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ vmbuffer, VISIBILITYMAP_VALID_BITS,
+ InvalidXLogRecPtr);
+ PageClearAllVisible(BufferGetPage(buffer));
}
/*
@@ -2161,13 +2165,22 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (all_visible_cleared)
+ XLogRegisterBuffer(HEAP_INSERT_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, info);
PageSetLSN(page, recptr);
+
+ if (all_visible_cleared)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -2441,10 +2454,12 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
- PageClearAllVisible(page);
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
visibilitymap_clear(relation->rd_locator,
BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ vmbuffer, VISIBILITYMAP_VALID_BITS,
+ InvalidXLogRecPtr);
+ PageClearAllVisible(page);
}
else if (all_frozen_set)
{
@@ -2577,7 +2592,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogRegisterData(xlrec, tupledata - scratch.data);
XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_HEAP, buffer,
REGBUF_STANDARD | bufflags);
- if (all_frozen_set)
+ if (all_frozen_set || all_visible_cleared)
XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_VM, vmbuffer, 0);
XLogRegisterBufData(HEAP_MULTI_INSERT_BLKREF_HEAP, tupledata,
@@ -2589,7 +2604,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
recptr = XLogInsert(RM_HEAP2_ID, info);
PageSetLSN(page, recptr);
- if (all_frozen_set)
+ if (all_frozen_set || all_visible_cleared)
{
Assert(BufferIsDirty(vmbuffer));
PageSetLSN(BufferGetPage(vmbuffer), recptr);
@@ -2598,7 +2613,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
END_CRIT_SECTION();
- if (all_frozen_set)
+ if (all_frozen_set || all_visible_cleared)
LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
UnlockReleaseBuffer(buffer);
@@ -3001,9 +3016,11 @@ l1:
if (PageIsAllVisible(page))
{
all_visible_cleared = true;
- PageClearAllVisible(page);
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ vmbuffer, VISIBILITYMAP_VALID_BITS,
+ InvalidXLogRecPtr);
+ PageClearAllVisible(page);
}
/* store transaction information of xact deleting the tuple */
@@ -3094,13 +3111,23 @@ l1:
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (all_visible_cleared)
+ XLogRegisterBuffer(HEAP_DELETE_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE);
PageSetLSN(page, recptr);
+
+ if (all_visible_cleared)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
if (vmbuffer != InvalidBuffer)
@@ -3239,6 +3266,8 @@ heap_update(Relation relation, const ItemPointerData *otid, HeapTuple newtup,
bool key_intact;
bool all_visible_cleared = false;
bool all_visible_cleared_new = false;
+ bool modified_vmbuffer = false;
+ bool modified_vmbuffer_new = false;
bool checked_lockers;
bool locker_remains;
bool id_has_external = false;
@@ -3347,8 +3376,11 @@ heap_update(Relation relation, const ItemPointerData *otid, HeapTuple newtup,
UnlockReleaseBuffer(buffer);
Assert(!have_tuple_lock);
- if (vmbuffer != InvalidBuffer)
+ if (BufferIsValid(vmbuffer))
+ {
ReleaseBuffer(vmbuffer);
+ vmbuffer = InvalidBuffer;
+ }
tmfd->ctid = *otid;
tmfd->xmax = InvalidTransactionId;
tmfd->cmax = InvalidCommandId;
@@ -3651,8 +3683,11 @@ l2:
UnlockReleaseBuffer(buffer);
if (have_tuple_lock)
UnlockTupleTuplock(relation, &(oldtup.t_self), *lockmode);
- if (vmbuffer != InvalidBuffer)
+ if (BufferIsValid(vmbuffer))
+ {
ReleaseBuffer(vmbuffer);
+ vmbuffer = InvalidBuffer;
+ }
*update_indexes = TU_None;
bms_free(hot_attrs);
@@ -3835,10 +3870,15 @@ l2:
* overhead would be unchanged, that doesn't seem necessarily
* worthwhile.
*/
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation->rd_locator, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear(relation->rd_locator, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN, InvalidXLogRecPtr))
+ cleared_all_frozen = true;
+ else
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ }
MarkBufferDirty(buffer);
@@ -3857,12 +3897,23 @@ l2:
xlrec.flags =
cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (cleared_all_frozen)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
/*
@@ -4063,20 +4114,89 @@ l2:
/* record address of new tuple in t_ctid of old one */
oldtup.t_data->t_ctid = heaptup->t_self;
- /* clear PD_ALL_VISIBLE flags, reset all visibilitymap bits */
- if (PageIsAllVisible(page))
+ /*
+ * Clear PD_ALL_VISIBLE flags and reset visibility map bits for any heap
+ * pages that were all-visible. If there are two heap pages, we may need
+ * to clear VM bits for both.
+ */
+ if (PageIsAllVisible(page) &&
+ newbuf != buffer && PageIsAllVisible(newpage) &&
+ vmbuffer_new == vmbuffer)
{
- all_visible_cleared = true;
+ /*
+ * This is the more complicated case: both the new and old heap pages
+ * are all-visible and both their VM bits are on the same page of the
+ * VM, so we register a single VM buffer as HEAP_UPDATE_BLKREF_VM_NEW
+ * in the WAL record. We must be careful to only lock and register one
+ * buffer, even though we modify it twice -- once for each heap
+ * block's VM bits.
+ */
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear(relation->rd_locator, block,
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS,
+ InvalidXLogRecPtr);
+ visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(newbuf),
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS,
+ InvalidXLogRecPtr);
PageClearAllVisible(page);
- visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ PageClearAllVisible(newpage);
+ modified_vmbuffer_new = true;
+ all_visible_cleared = true;
+ all_visible_cleared_new = true;
}
- if (newbuf != buffer && PageIsAllVisible(newpage))
+ else
{
- all_visible_cleared_new = true;
- PageClearAllVisible(newpage);
- visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(newbuf),
- vmbuffer_new, VISIBILITYMAP_VALID_BITS);
+ /*
+ * In all the remaining cases, we will clear at most one heap block's
+ * VM bits per VM page.
+ */
+ bool old_all_visible = PageIsAllVisible(page);
+ bool new_all_visible = newbuf != buffer && PageIsAllVisible(newpage);
+ Buffer vmbuffers[2] = {
+ old_all_visible ? vmbuffer : InvalidBuffer,
+ new_all_visible ? vmbuffer_new : InvalidBuffer
+ };
+
+ /*
+ * When both pages need different VM pages cleared, acquire the VM
+ * buffer locks in VM block order to avoid deadlocks between backends
+ * updating tuples in opposite directions across VM pages.
+ */
+ if (old_all_visible && new_all_visible &&
+ BufferGetBlockNumber(vmbuffers[0]) > BufferGetBlockNumber(vmbuffers[1]))
+ {
+ Buffer t = vmbuffers[0];
+
+ vmbuffers[0] = vmbuffers[1];
+ vmbuffers[1] = t;
+ }
+
+ Assert((!BufferIsValid(vmbuffers[0]) && !BufferIsValid(vmbuffers[1])) ||
+ vmbuffers[0] != vmbuffers[1]);
+
+ if (BufferIsValid(vmbuffers[0]))
+ LockBuffer(vmbuffers[0], BUFFER_LOCK_EXCLUSIVE);
+ if (BufferIsValid(vmbuffers[1]))
+ LockBuffer(vmbuffers[1], BUFFER_LOCK_EXCLUSIVE);
+
+ if (old_all_visible)
+ {
+ visibilitymap_clear(relation->rd_locator, block,
+ vmbuffer, VISIBILITYMAP_VALID_BITS,
+ InvalidXLogRecPtr);
+ modified_vmbuffer = true;
+ PageClearAllVisible(page);
+ all_visible_cleared = true;
+ }
+ if (new_all_visible)
+ {
+ visibilitymap_clear(relation->rd_locator, BufferGetBlockNumber(newbuf),
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS,
+ InvalidXLogRecPtr);
+ modified_vmbuffer_new = true;
+ PageClearAllVisible(newpage);
+ all_visible_cleared_new = true;
+ }
}
if (newbuf != buffer)
@@ -4099,20 +4219,31 @@ l2:
}
recptr = log_heap_update(relation, buffer,
- newbuf, &oldtup, heaptup,
+ modified_vmbuffer ? vmbuffer : InvalidBuffer,
+ newbuf,
+ modified_vmbuffer_new ? vmbuffer_new : InvalidBuffer,
+ &oldtup, heaptup,
old_key_tuple,
all_visible_cleared,
all_visible_cleared_new,
walLogical);
if (newbuf != buffer)
- {
PageSetLSN(newpage, recptr);
- }
PageSetLSN(page, recptr);
+
+ if (modified_vmbuffer)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
+ if (modified_vmbuffer_new)
+ PageSetLSN(BufferGetPage(vmbuffer_new), recptr);
}
END_CRIT_SECTION();
+ if (modified_vmbuffer)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ if (modified_vmbuffer_new)
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_UNLOCK);
+
if (newbuf != buffer)
LockBuffer(newbuf, BUFFER_LOCK_UNLOCK);
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
@@ -5161,10 +5292,16 @@ failed:
tuple->t_data->t_ctid = *tid;
/* Clear only the all-frozen bit on visibility map if needed */
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation->rd_locator, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear(relation->rd_locator, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN,
+ InvalidXLogRecPtr))
+ cleared_all_frozen = true;
+ else
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ }
MarkBufferDirty(*buffer);
@@ -5196,15 +5333,25 @@ failed:
xlrec.flags = cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
/* we don't decode row locks atm, so no need to log the origin */
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (cleared_all_frozen)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
result = TM_Ok;
out_locked:
@@ -5915,10 +6062,16 @@ l4:
xid, mode, false,
&new_xmax, &new_infomask, &new_infomask2);
- if (PageIsAllVisible(BufferGetPage(buf)) &&
- visibilitymap_clear(rel->rd_locator, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(BufferGetPage(buf)))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear(rel->rd_locator, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN,
+ InvalidXLogRecPtr))
+ cleared_all_frozen = true;
+ else
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ }
START_CRIT_SECTION();
@@ -5949,13 +6102,23 @@ l4:
XLogRegisterData(&xlrec, SizeOfHeapLockUpdated);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_LOCK_UPDATED);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (cleared_all_frozen)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
next:
/* if we find the end of update chain, we're done. */
if (mytup.t_data->t_infomask & HEAP_XMAX_INVALID ||
@@ -8776,8 +8939,9 @@ bottomup_sort_and_shrink(TM_IndexDeleteOp *delstate)
* have modified the buffer(s) and marked them dirty.
*/
static XLogRecPtr
-log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup, HeapTuple newtup,
+log_heap_update(Relation reln, Buffer oldbuf, Buffer vmbuffer_old,
+ Buffer newbuf, Buffer vmbuffer_new,
+ HeapTuple oldtup, HeapTuple newtup,
HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared,
bool walLogical)
@@ -8987,6 +9151,20 @@ log_heap_update(Relation reln, Buffer oldbuf,
old_key_tuple->t_len - SizeofHeapTupleHeader);
}
+ /*
+ * Register VM buffers. If the old and new heap pages' VM bits are on the
+ * same VM page, the caller passes only vmbuffer_new (mirroring the heap
+ * page convention where block 0 = new is always registered).
+ */
+ Assert((BufferIsInvalid(vmbuffer_old) && BufferIsInvalid(vmbuffer_new)) ||
+ (vmbuffer_old != vmbuffer_new));
+
+ if (BufferIsValid(vmbuffer_new))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_NEW, vmbuffer_new, 0);
+
+ if (BufferIsValid(vmbuffer_old))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_OLD, vmbuffer_old, 0);
+
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index 4e994cd01f3..7e55fda5603 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -310,13 +310,15 @@ heap_xlog_delete(XLogReaderState *record)
*/
if (xlrec->flags & XLH_DELETE_ALL_VISIBLE_CLEARED)
{
- Relation reln = CreateFakeRelcacheEntry(target_locator);
Buffer vmbuffer = InvalidBuffer;
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(target_locator, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ if (XLogReadBufferForRedo(record, HEAP_DELETE_BLKREF_VM,
+ &vmbuffer) == BLK_NEEDS_REDO)
+ visibilitymap_clear(target_locator,
+ blkno, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
}
if (XLogReadBufferForRedo(record, HEAP_DELETE_BLKREF_HEAP,
@@ -399,13 +401,14 @@ heap_xlog_insert(XLogReaderState *record)
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
{
- Relation reln = CreateFakeRelcacheEntry(target_locator);
Buffer vmbuffer = InvalidBuffer;
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(target_locator, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ if (XLogReadBufferForRedo(record, HEAP_INSERT_BLKREF_VM,
+ &vmbuffer) == BLK_NEEDS_REDO)
+ visibilitymap_clear(target_locator, blkno, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
}
/*
@@ -528,18 +531,24 @@ heap_xlog_multi_insert(XLogReaderState *record)
(xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
/*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
+ * If required, clear the VM first, to prevent all-visible temporarily
+ * being set for a heap page that's not all visible anymore.
+ *
+ * If it were possible for XLH_INSERT_ALL_VISIBLE_CLEARED and
+ * XLH_INSERT_ALL_FROZEN_SET to be present in the same record, doing the
+ * XLogReadBufferForRedo() before the PageSetAllVisible() below would be a
+ * problem, as it'd violate the rule that a heap page must never be set
+ * all-visible in the VM while its PD_ALL_VISIBLE is clear.
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
{
- Relation reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(rlocator, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
+ if (XLogReadBufferForRedo(record, HEAP_MULTI_INSERT_BLKREF_VM,
+ &vmbuffer) == BLK_NEEDS_REDO)
+ visibilitymap_clear(rlocator, blkno, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
vmbuffer = InvalidBuffer;
- FreeFakeRelcacheEntry(reln);
}
if (isinit)
@@ -640,7 +649,7 @@ heap_xlog_multi_insert(XLogReaderState *record)
buffer = InvalidBuffer;
/*
- * Read and update the visibility map (VM) block.
+ * Read and update the visibility map (VM) block to set it frozen.
*
* We must always redo VM changes, even if the corresponding heap page
* update was skipped due to the LSN interlock. Each VM block covers
@@ -713,6 +722,9 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
nbuffer;
Page opage,
npage;
+ bool new_cleared,
+ old_cleared;
+ bool same_vm_page;
OffsetNumber offnum;
ItemId lp;
HeapTupleData oldtup;
@@ -752,16 +764,64 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
*/
- if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED)
+ new_cleared = (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0;
+ old_cleared = (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) != 0;
+
+ /*
+ * If both the old and new heap pages were all-visible and their VM bits
+ * are on the same VM page, that single VM page is registered as
+ * HEAP_UPDATE_BLKREF_VM_NEW. Clear both heap blocks' VM bits from the
+ * single provided VM buffer.
+ */
+ same_vm_page = new_cleared && old_cleared &&
+ !XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_OLD) &&
+ XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_NEW);
+
+ if (same_vm_page)
{
- Relation reln = CreateFakeRelcacheEntry(rlocator);
Buffer vmbuffer = InvalidBuffer;
- visibilitymap_pin(reln, oldblk, &vmbuffer);
- visibilitymap_clear(rlocator, oldblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW,
+ &vmbuffer) == BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear(rlocator, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ visibilitymap_clear(rlocator, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ }
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
}
+ else
+ {
+ /*
+ * We can be sure that we need to clear at most one heap page's VM
+ * bits in each registered VM buffer.
+ */
+ if (new_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW,
+ &vmbuffer) == BLK_NEEDS_REDO)
+ visibilitymap_clear(rlocator, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ if (old_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_OLD,
+ &vmbuffer) == BLK_NEEDS_REDO)
+ visibilitymap_clear(rlocator, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ }
+
/*
* In normal operation, it is important to lock the two pages in
@@ -834,21 +894,6 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
newaction = XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_NEW,
&nbuffer);
- /*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
- */
- if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, newblk, &vmbuffer);
- visibilitymap_clear(rlocator, newblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
-
/* Deal with new tuple */
if (newaction == BLK_NEEDS_REDO)
{
@@ -1043,17 +1088,16 @@ heap_xlog_lock(XLogReaderState *record)
RelFileLocator rlocator;
Buffer vmbuffer = InvalidBuffer;
BlockNumber block;
- Relation reln;
XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
&block);
- reln = CreateFakeRelcacheEntry(rlocator);
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(rlocator, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
-
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_VM,
+ &vmbuffer) == BLK_NEEDS_REDO)
+ visibilitymap_clear(rlocator, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN,
+ lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
}
if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
@@ -1121,17 +1165,16 @@ heap_xlog_lock_updated(XLogReaderState *record)
RelFileLocator rlocator;
Buffer vmbuffer = InvalidBuffer;
BlockNumber block;
- Relation reln;
XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
&block);
- reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(rlocator, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_VM,
+ &vmbuffer) == BLK_NEEDS_REDO)
+ visibilitymap_clear(rlocator, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN,
+ lsn);
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
}
if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c
index 184fb70a24f..4100fe42d40 100644
--- a/src/backend/access/heap/pruneheap.c
+++ b/src/backend/access/heap/pruneheap.c
@@ -929,7 +929,7 @@ heap_page_fix_vm_corruption(PruneState *prstate, OffsetNumber offnum,
{
visibilitymap_clear(prstate->relation->rd_locator, prstate->block,
prstate->vmbuffer,
- VISIBILITYMAP_VALID_BITS);
+ VISIBILITYMAP_VALID_BITS, InvalidXLogRecPtr);
prstate->old_vmbits = 0;
}
}
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 3b699291b23..03f12a4b180 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -139,22 +139,26 @@
static Buffer vm_readbuf(Relation rel, BlockNumber blkno, bool extend);
static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks);
-
/*
* visibilitymap_clear - clear specified bits for one page in visibility map
*
- * You must pass a buffer containing the correct map page to this function.
- * Call visibilitymap_pin first to pin the right one. This function doesn't do
- * any I/O. Returns true if any bits have been cleared and false otherwise.
+ * You must pass a buffer containing the correct map page to this function,
+ * which already needs to be pinned and locked exclusively.
+ *
+ * This function doesn't do any I/O. Returns true if any bits have been
+ * cleared and false otherwise.
*/
bool
-visibilitymap_clear(RelFileLocator rlocator,
- BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
+visibilitymap_clear(RelFileLocator rlocator, BlockNumber heapBlk,
+ Buffer vmbuf, uint8 flags, XLogRecPtr lsn)
{
- BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
int mapOffset = HEAPBLK_TO_OFFSET(heapBlk);
+#ifdef USE_ASSERT_CHECKING
+ BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
+#endif
uint8 mask = flags << mapOffset;
+ Page page;
char *map;
bool cleared = false;
@@ -168,11 +172,11 @@ visibilitymap_clear(RelFileLocator rlocator,
heapBlk);
#endif
- if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
- elog(ERROR, "wrong buffer passed to visibilitymap_clear");
+ Assert(BufferIsValid(vmbuf) && BufferGetBlockNumber(vmbuf) == mapBlock);
+ Assert(BufferIsLockedByMeInMode(vmbuf, BUFFER_LOCK_EXCLUSIVE));
- LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
- map = PageGetContents(BufferGetPage(vmbuf));
+ page = BufferGetPage(vmbuf);
+ map = PageGetContents(page);
if (map[mapByte] & mask)
{
@@ -182,7 +186,15 @@ visibilitymap_clear(RelFileLocator rlocator,
cleared = true;
}
- LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
+ if (XLogRecPtrIsValid(lsn))
+ {
+ /*
+ * Outside of recovery we won't have an LSN to stamp, as we make page
+ * changes before emitting WAL
+ */
+ Assert(InRecovery);
+ PageSetLSN(page, lsn);
+ }
return cleared;
}
diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl
index f5fe94f9f15..bebc1137c0f 100644
--- a/src/bin/pg_walsummary/t/002_blocks.pl
+++ b/src/bin/pg_walsummary/t/002_blocks.pl
@@ -93,13 +93,14 @@ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
split(m@/@, $end_lsn);
ok(-f $filename, "WAL summary file exists");
-# Run pg_walsummary on it. We expect exactly two blocks to be modified,
-# block 0 and one other.
+# Run pg_walsummary on it. We expect exactly three blocks to be modified,
+# block 0 (old tuple), another block (new tuple), and the block for the VM.
my ($stdout, $stderr) = run_command([ 'pg_walsummary', '-i', $filename ]);
note($stdout);
@lines = split(/\n/, $stdout);
like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified");
+like($stdout, qr/FORK vm: block 0$/m, "stdout shows VM block 0 modified");
is($stderr, '', 'stderr is empty');
-is(0 + @lines, 2, "UPDATE modified 2 blocks");
+is(0 + @lines, 3, "UPDATE modified 3 blocks");
done_testing();
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index d425562fd79..9c3d3945b08 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -227,10 +227,19 @@ typedef struct xl_multi_insert_tuple
*
* HEAP_UPDATE_BLKREF_OLD: old page, if different. (no data, just a reference
* to the block)
+ *
+ * HEAP_UPDATE_BLKREF_VM_NEW: VM page covering the new heap page. Registered
+ * whenever XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED is set. Also covers the old heap
+ * page's VM bits when both heap pages map to the same VM page.
+ *
+ * HEAP_UPDATE_BLKREF_VM_OLD: VM page covering the old heap page. Only
+ * registered when XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED is set and the old heap
+ * page's VM bits are on a different VM page from the new heap page's.
*/
-
-#define HEAP_UPDATE_BLKREF_NEW 0
-#define HEAP_UPDATE_BLKREF_OLD 1
+#define HEAP_UPDATE_BLKREF_NEW 0
+#define HEAP_UPDATE_BLKREF_OLD 1
+#define HEAP_UPDATE_BLKREF_VM_NEW 2
+#define HEAP_UPDATE_BLKREF_VM_OLD 3
typedef struct xl_heap_update
{
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index b860c4ef3ca..9123590efda 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -15,6 +15,7 @@
#define VISIBILITYMAP_H
#include "access/visibilitymapdefs.h"
+#include "access/xlogdefs.h"
#include "storage/block.h"
#include "storage/buf.h"
#include "storage/relfilelocator.h"
@@ -27,7 +28,7 @@
((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0)
extern bool visibilitymap_clear(RelFileLocator rlocator, BlockNumber heapBlk,
- Buffer vmbuf, uint8 flags);
+ Buffer vmbuf, uint8 flags, XLogRecPtr lsn);
extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk,
Buffer *vmbuf);
extern bool visibilitymap_pin_ok(BlockNumber heapBlk, Buffer vmbuf);
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 13ae3ad4fbb..55663e6f4af 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -32,7 +32,7 @@
/*
* Each page of XLOG file has a header like this:
*/
-#define XLOG_PAGE_MAGIC 0xD11F /* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD120 /* can be used as WAL version indicator */
typedef struct XLogPageHeaderData
{
--
2.43.0
v2_PG19-0004-Put-pg_combinebackup-debug-test-output-in-te.patch (text/x-patch, 4.2K) download
From 3dd83aa7bec56a8acc24cffc4b0807fd131c5e06 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 30 Apr 2026 16:06:06 -0400
Subject: [PATCH v2_PG19 4/8] Put pg_combinebackup debug test output in temp
file
Having --debug as the default option to pg_combinebackup in
init_from_backup() makes the logs extremely verbose. It is used for
debugging failures, so put the output in a temp file and only keep it
around if tests fail.
Author: Melanie Plageman <melanieplageman@gmail.com>
Disussion: https://postgr.es/m/y73coty2smkxgvelh4e32kqrt5pyn2pb7cz7x4mrz6yaxitqd5%40nzhkeazqf4bn
Backpatch-through: 17
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 83 +++++++++++++++++++++++-
1 file changed, 81 insertions(+), 2 deletions(-)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index bdc051e6ead..766d1621c09 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -125,6 +125,8 @@ our $min_compat = 12;
# list of file reservations made by get_free_port
my @port_reservation_files;
+my @combinebackup_debug_logs;
+my %combinebackup_debug_logs_printed;
# We want to choose a server port above the range that servers typically use
# on Unix systems and below the range those systems typically use for ephemeral
@@ -925,7 +927,9 @@ pathnames that should be used for the target cluster.
To restore from an incremental backup, pass the parameter combine_with_prior
as a reference to an array of prior backup names with which this backup
-is to be combined using pg_combinebackup.
+is to be combined using pg_combinebackup. pg_combinebackup is run with
+--debug, but the output is captured in a temporary file and only copied to the
+test log if pg_combinebackup fails or if a later test fails.
Streaming replication can be enabled on this node by passing the keyword
parameter has_streaming => 1. This is disabled by default.
@@ -991,7 +995,7 @@ sub init_from_backup
}
push @combineargs, @prior_backup_path, $backup_path,
'--output' => $data_path;
- PostgreSQL::Test::Utils::system_or_bail(@combineargs);
+ _run_pg_combinebackup_with_debug_log(@combineargs);
}
elsif (defined $params{tar_program})
{
@@ -1106,6 +1110,81 @@ port = $port
return;
}
+sub _run_pg_combinebackup_with_debug_log
+{
+ my (@cmd) = @_;
+ my $debug_log = PostgreSQL::Test::Utils::tempdir('pg_combinebackup') .
+ '/debug.log';
+
+ push @combinebackup_debug_logs, $debug_log;
+
+ print("# Running: " . join(" ", @cmd) . "\n");
+ # If this pg_combinebackup invocation itself fails, print its captured
+ # output immediately. Successful invocations may still be printed later if
+ # some subsequent TAP assertion in this test script fails.
+ if (!IPC::Run::run(\@cmd, '&>' => $debug_log))
+ {
+ _print_pg_combinebackup_debug_log($debug_log);
+
+ if ($? == -1)
+ {
+ BAIL_OUT(
+ sprintf(
+ "failed to execute command \"%s\": $!", join(" ", @cmd)));
+ }
+ elsif ($? & 127)
+ {
+ BAIL_OUT(
+ sprintf(
+ "command \"%s\" died with signal %d",
+ join(" ", @cmd), $? & 127));
+ }
+ else
+ {
+ BAIL_OUT(
+ sprintf(
+ "command \"%s\" exited with value %d",
+ join(" ", @cmd), $? >> 8));
+ }
+ }
+}
+
+sub _print_pg_combinebackup_debug_log
+{
+ my ($debug_log) = @_;
+
+ return unless -e $debug_log;
+ return if $combinebackup_debug_logs_printed{$debug_log}++;
+
+ diag("pg_combinebackup debug output from $debug_log:\n" .
+ PostgreSQL::Test::Utils::slurp_file($debug_log));
+}
+
+# Keep pg_combinebackup debug logs until test exit. If pg_combinebackup itself
+# fails, its log is printed at the failure site above. If pg_combinebackup
+# succeeds but a later TAP assertion fails, print the retained logs here so the
+# regress output includes the combine step that produced the restored state.
+# Remove the logs after a clean run.
+END
+{
+ # take care not to change the script's exit value
+ my $exit_code = $?;
+
+ if ($exit_code == 0 && PostgreSQL::Test::Utils::all_tests_passing())
+ {
+ unlink @combinebackup_debug_logs;
+ }
+ else
+ {
+ foreach my $debug_log (@combinebackup_debug_logs)
+ {
+ _print_pg_combinebackup_debug_log($debug_log);
+ }
+ }
+
+ $? = $exit_code;
+}
+
=pod
=item $node->rotate_logfile()
--
2.43.0
v2_PG19-0005-Test-that-VM-clear-registers-VM-buffers.patch (text/x-patch, 7.9K) download
From 318b714966534c3dbac489574fb0db6ef7d3f450 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 23 Apr 2026 11:13:23 -0400
Subject: [PATCH v2_PG19 5/8] Test that VM clear registers VM buffers
WAL summarizer only includes registered buffers, so you could end up
with data corruption in incremental backups if VM clears were meant to
be covered by the incremental backup.
Author: Melanie Plageman <melanieplageman@gmail.com>
Discussion: https://postgr.es/m/oqcsevg35xjan2327x5kdfth6q4fgeqboxfo3v3imeyih2uiny%406sez5dzxl6nt
Backpatch-through: 17
---
src/bin/pg_combinebackup/meson.build | 1 +
.../pg_combinebackup/t/012_vm_consistency.pl | 205 ++++++++++++++++++
2 files changed, 206 insertions(+)
create mode 100644 src/bin/pg_combinebackup/t/012_vm_consistency.pl
diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build
index a35b86f3f59..ba1c8cfa3d0 100644
--- a/src/bin/pg_combinebackup/meson.build
+++ b/src/bin/pg_combinebackup/meson.build
@@ -39,6 +39,7 @@ tests += {
't/009_no_full_file.pl',
't/010_hardlink.pl',
't/011_ib_truncation.pl',
+ 't/012_vm_consistency.pl',
],
}
}
diff --git a/src/bin/pg_combinebackup/t/012_vm_consistency.pl b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
new file mode 100644
index 00000000000..7b9c577d19a
--- /dev/null
+++ b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
@@ -0,0 +1,205 @@
+# Copyright (c) 2021-2026, PostgreSQL Global Development Group
+#
+# Test that heap operations clearing visibility map bits (INSERT, UPDATE,
+# DELETE, SELECT FOR UPDATE, COPY) correctly register visibility map buffers,
+# since incremental backups rely on the WAL summarizer, which only tracks
+# registered buffers.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir_short();
+my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
+
+# Set up primary with WAL summarization enabled.
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(has_archiving => 1, allows_streaming => 1);
+$primary->append_conf('postgresql.conf', <<EOF);
+summarize_wal = on
+autovacuum = off
+EOF
+$primary->start;
+
+$primary->safe_psql('postgres', q{CREATE EXTENSION pg_visibility});
+
+my @tests = (
+ {
+ label => 'INSERT',
+ table => 'vm_insert_test',
+ setup => q{CREATE TABLE vm_insert_test (id int);
+ INSERT INTO vm_insert_test DEFAULT VALUES;},
+ modify => q{INSERT INTO vm_insert_test VALUES (1)},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'DELETE',
+ table => 'vm_delete_test',
+ setup => q{CREATE TABLE vm_delete_test (id int);
+ INSERT INTO vm_delete_test VALUES (1), (2);},
+ modify => q{DELETE FROM vm_delete_test WHERE id = 1},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'UPDATE',
+ table => 'vm_update_test',
+ # Include both same-page and cross-page updates.
+ setup => q{CREATE TABLE vm_update_test (id INT, val TEXT);
+ INSERT INTO vm_update_test VALUES (1, 'same page'), (2, 'cross page');
+ INSERT INTO vm_update_test SELECT i, repeat('a', 200)
+ FROM generate_series(3, 70) i;},
+ modify => q{UPDATE vm_update_test SET id = 0 WHERE id = 1;
+ UPDATE vm_update_test SET val = repeat('b', 8000) WHERE id = 2;},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'LOCK',
+ table => 'vm_lock_test',
+ setup => q{CREATE TABLE vm_lock_test (id int);
+ INSERT INTO vm_lock_test VALUES (1), (2);},
+ modify => q{SELECT * FROM vm_lock_test WHERE id = 1 FOR UPDATE},
+ visible_op => '==',
+ frozen_op => '<',
+ },
+ {
+ label => 'COPY',
+ table => 'vm_copy_test',
+ setup => q{CREATE TABLE vm_copy_test (id int);
+ INSERT INTO vm_copy_test DEFAULT VALUES;},
+ modify => q{COPY vm_copy_test FROM PROGRAM 'echo 42'},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+);
+
+sub get_vm_summary
+{
+ my ($node, $table) = @_;
+ my $result = $node->safe_psql('postgres',
+ "SELECT all_visible, all_frozen FROM pg_visibility_map_summary('$table')");
+ my @vals = split(/\|/, $result);
+ return @vals;
+}
+
+# Confirm VACUUM (FREEZE) set VM bits before testing whether later heap
+# modifications clear those bits and are captured by incremental backup.
+sub check_vacuumed_vm
+{
+ my ($node, $test) = @_;
+ my ($all_visible, $all_frozen) = get_vm_summary($node, $test->{table});
+
+ cmp_ok($all_visible, '>', 0,
+ "$test->{label} test: pages are all-visible after vacuum");
+ cmp_ok($all_frozen, '>', 0,
+ "$test->{label} test: pages are all-frozen after vacuum");
+
+ return ($all_visible, $all_frozen);
+}
+
+# Check the VM bit counts after a heap modification against the post-vacuum
+# baseline. Most operations clear both bits; tuple locking clears all-frozen
+# without clearing all-visible.
+sub check_modified_vm
+{
+ my ($node, $test) = @_;
+ my ($post_visible, $post_frozen) = get_vm_summary($node, $test->{table});
+
+ cmp_ok($post_visible, $test->{visible_op}, $test->{pre_visible},
+ "$test->{label} test: all-visible state after modification");
+ cmp_ok($post_frozen, $test->{frozen_op}, $test->{pre_frozen},
+ "$test->{label} test: all-frozen state after modification");
+
+ return ($post_visible, $post_frozen);
+}
+
+# Verify the combined backup restored VM state exactly as it exists on the
+# primary, and ask pg_visibility to check that visible tuples are consistent.
+sub validate_restored_vm
+{
+ my ($restored, $test) = @_;
+
+ my ($primary_visible, $primary_frozen) =
+ get_vm_summary($primary, $test->{table});
+ my ($restored_visible, $restored_frozen) =
+ get_vm_summary($restored, $test->{table});
+
+ is($restored_visible, $primary_visible,
+ "$test->{label} test: restored all_visible count matches primary");
+ is($restored_frozen, $primary_frozen,
+ "$test->{label} test: restored all_frozen count matches primary");
+
+ my $corrupt_tids = $restored->safe_psql('postgres',
+ "SELECT count(*) FROM pg_check_visible('$test->{table}')");
+ is($corrupt_tids, '0',
+ "$test->{label} test: no VM corruption detected by pg_check_visible");
+}
+
+# Create and populate the tables, then vacuum freeze them to set the VM bits
+foreach my $test (@tests)
+{
+ $primary->safe_psql('postgres', $test->{setup});
+ $primary->safe_psql('postgres', "VACUUM (FREEZE) $test->{table}");
+ ($test->{pre_visible}, $test->{pre_frozen}) =
+ check_vacuumed_vm($primary, $test);
+}
+
+# Take a full backup
+my $full_name = 'full';
+my $full_path = $primary->backup_dir . "/$full_name";
+$primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $full_path,
+ '--checkpoint' => 'fast',
+ ],
+ 'full backup');
+
+# Modify the tables and check that the VM bits are as expected for that test
+# after the specified modification.
+foreach my $test (@tests)
+{
+ $primary->safe_psql('postgres', $test->{modify});
+ ($test->{post_visible}, $test->{post_frozen}) =
+ check_modified_vm($primary, $test);
+}
+
+# Take an incremental backup. This will have the changes made in the
+# modification step.
+my $incr_name = 'incr';
+my $incr_path = $primary->backup_dir . "/$incr_name";
+$primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $incr_path,
+ '--checkpoint' => 'fast',
+ '--incremental' => $full_path . '/backup_manifest',
+ ],
+ 'incremental backup');
+
+# Start a server from a combined backup composed of the incremental and full
+# backup.
+my $restored = PostgreSQL::Test::Cluster->new('restored');
+$restored->init_from_backup($primary, $incr_name,
+ combine_with_prior => [$full_name],
+ combine_mode => $mode);
+$restored->append_conf('postgresql.conf', <<EOF);
+autovacuum = off
+EOF
+$restored->start;
+$restored->safe_psql('postgres', q{CREATE EXTENSION IF NOT EXISTS pg_visibility});
+
+# Confirm that the restored server's visibility map matches the original server
+foreach my $test (@tests)
+{
+ validate_restored_vm($restored, $test);
+}
+
+$restored->stop;
+$primary->stop;
+
+done_testing();
--
2.43.0
v2_PG19-0006-Add-VM-corruption-check-to-verify_heapam.patch (text/x-patch, 2.2K) download
From b6ddc887453446dafc28782d876517293b2b8a88 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 10:20:20 -0400
Subject: [PATCH v2_PG19 6/8] Add VM corruption check to verify_heapam()
pg_check_visible() can catch cases were tuples are not visible to all
and the VM is set, but it doesn't specifically check page hints. This
doesn't check, for example, an INSERT failing to clear PD_ALL_VISIBLE.
This also helps users who have access to amcheck but not pg_visibility
to diagnose corruption.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/flat/CAAKRu_bn%2Be7F4yPFBgFbnP%2BsyJRKyNK092bjD2LKvZW7O4Svag%40mail.gmail.com
---
contrib/amcheck/verify_heapam.c | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 20ff58aa782..823f05e5916 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -481,6 +481,7 @@ verify_heapam(PG_FUNCTION_ARGS)
while ((ctx.buffer = read_stream_next_buffer(stream, NULL)) != InvalidBuffer)
{
+ uint8 vmbits;
OffsetNumber maxoff;
OffsetNumber predecessor[MaxOffsetNumber];
OffsetNumber successor[MaxOffsetNumber];
@@ -499,6 +500,24 @@ verify_heapam(PG_FUNCTION_ARGS)
ctx.blkno = BufferGetBlockNumber(ctx.buffer);
ctx.page = BufferGetPage(ctx.buffer);
+ /*
+ * It is corruption if the page-level PD_ALL_VISIBLE flag is clear and
+ * the visibility map bits corresponding to this heap page are set.
+ * Any corruption on the VM page itself won't be caught, and won't
+ * cause visibilitymap_get_status() to fail, because the VM page is
+ * read with ZERO_ON_ERROR.
+ */
+ vmbits = visibilitymap_get_status(ctx.rel, ctx.blkno, &vmbuffer);
+
+ if (!PageIsAllVisible(ctx.page) &&
+ (vmbits & VISIBILITYMAP_VALID_BITS) != 0)
+ {
+ ctx.offnum = InvalidOffsetNumber;
+ ctx.attnum = -1;
+ report_corruption(&ctx,
+ psprintf("page is not marked all-visible in page header but visibility map bit is set"));
+ }
+
/* Perform tuple checks */
maxoff = PageGetMaxOffsetNumber(ctx.page);
for (ctx.offnum = FirstOffsetNumber; ctx.offnum <= maxoff;
--
2.43.0
v2_PG19-0007-Stop-masking-PD_ALL_VISIBLE-when-checking-WA.patch (text/x-patch, 1.6K) download
From 1d2945bb01e68158eb812928301b1fed44ddb33e Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 10:33:46 -0400
Subject: [PATCH v2_PG19 7/8] Stop masking PD_ALL_VISIBLE when checking WAL
consistency
Setting and clearing PD_ALL_VISIBLE is WAL-logged, so there is no need
to mask it for the purposes of WAL consistency checking.
There is one minor exception, which is that when fixing VM corruption,
we clear PD_ALL_VISIBLE on the heap page without specifically emitting
WAL. This is a situation where corruption has already happened, however,
so it is okay if WAL consistency checking flags this as well.
Stop masking PD_ALL_VISIBLE, as doing so can only conceal real bugs.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/flat/CAAKRu_bn%2Be7F4yPFBgFbnP%2BsyJRKyNK092bjD2LKvZW7O4Svag%40mail.gmail.com
---
src/backend/access/common/bufmask.c | 6 ------
1 file changed, 6 deletions(-)
diff --git a/src/backend/access/common/bufmask.c b/src/backend/access/common/bufmask.c
index 5f63d04c9cb..93e34ea38b8 100644
--- a/src/backend/access/common/bufmask.c
+++ b/src/backend/access/common/bufmask.c
@@ -53,12 +53,6 @@ mask_page_hint_bits(Page page)
/* Ignore PD_PAGE_FULL and PD_HAS_FREE_LINES flags, they are just hints. */
PageClearFull(page);
PageClearHasFreeLinePointers(page);
-
- /*
- * PD_ALL_VISIBLE is masked during WAL consistency checking. XXX: It is
- * worth investigating if we could stop doing this.
- */
- PageClearAllVisible(page);
}
/*
--
2.43.0
v2_PG19-0008-xlog-Verify-all-block-references-are-used-du.patch (text/x-patch, 4.2K) download
From 6474a4b261ca9c78163e5b9679222a3837f9f383 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 12:08:26 -0400
Subject: [PATCH v2_PG19 8/8] xlog: Verify all block references are used during
replay
It's unclear if this makes sense to actually commit. It requires that
blocks read during recovery go through XLogReadBufferForRedoExtended()
and has a special case for handling XLOG_FPI_FOR_HINT when full page
writes are disabled.
Author: Andres Freund <andres@anarazel.de>
Co-authored-by: Melanie Plageman <melanieplageman@gmail.com>
Discussion: https://postgr.es/m/oqcsevg35xjan2327x5kdfth6q4fgeqboxfo3v3imeyih2uiny%406sez5dzxl6nt
---
src/backend/access/transam/xlog.c | 10 ++++++++++
src/backend/access/transam/xlogreader.c | 4 ++++
src/backend/access/transam/xlogrecovery.c | 16 ++++++++++++++++
src/backend/access/transam/xlogutils.c | 6 ++++++
src/include/access/xlogreader.h | 4 ++++
5 files changed, 40 insertions(+)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f0434da40c9..a8069ff729e 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9081,6 +9081,16 @@ xlog_redo(XLogReaderState *record)
{
if (info == XLOG_FPI)
elog(ERROR, "XLOG_FPI record did not contain a full-page image");
+
+#ifdef USE_ASSERT_CHECKING
+
+ /*
+ * If full_page_writes are disabled, we don't want to error
+ * out because we didn't read the block.
+ */
+ if (info == XLOG_FPI_FOR_HINT)
+ record->record->blocks[block_id].used_read = true;
+#endif
continue;
}
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 8849610db00..63747aec566 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1794,6 +1794,10 @@ DecodeXLogRecord(XLogReaderState *state,
blk = &decoded->blocks[block_id];
blk->in_use = true;
+#ifdef USE_ASSERT_CHECKING
+ blk->used_read = false;
+#endif
+
blk->apply_image = false;
COPY_HEADER_FIELD(&fork_flags, sizeof(uint8));
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 73b78a83fa7..7e7d0f57412 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -1965,6 +1965,22 @@ ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *repl
/* Now apply the WAL record itself */
GetRmgr(record->xl_rmid).rm_redo(xlogreader);
+ /*
+ * Verify that all block references were used during replay. This helps
+ * detect bugs in redo routines. Every referenced block needs to be
+ * replayed with XLogReadBufferForRedoExtended(), possibly via a helper,
+ * to ensure that we replay FPIs, extend the relation if necessary, and
+ * other perform other similar protections.
+ */
+#ifdef USE_ASSERT_CHECKING
+ for (int block_id = 0; block_id <= xlogreader->record->max_block_id; block_id++)
+ {
+ DecodedBkpBlock *blk = &xlogreader->record->blocks[block_id];
+
+ Assert(!blk->in_use || blk->used_read);
+ }
+#endif
+
/*
* After redo, check whether the backup pages associated with the WAL
* record are consistent with the existing pages. This check is done only
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 5fbe39133b8..979d6465159 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -359,6 +359,12 @@ XLogReadBufferForRedoExtended(XLogReaderState *record,
block_id);
}
+#ifdef USE_ASSERT_CHECKING
+ /* Shouldn't have multiple read references to block */
+ Assert(!record->record->blocks[block_id].used_read);
+ record->record->blocks[block_id].used_read = true;
+#endif
+
/*
* Make sure that if the block is marked with WILL_INIT, the caller is
* going to initialize it. And vice versa.
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 97eae2c1dab..fd2119005e8 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -145,6 +145,10 @@ typedef struct
bool has_data;
char *data;
uint16 data_len;
+
+#ifdef USE_ASSERT_CHECKING
+ bool used_read;
+#endif
} DecodedBkpBlock;
/*
--
2.43.0
v2_PG18-0001-Alias-WAL-block-references-for-some-record-t.patch (text/x-patch, 15.2K) download
From 260c0bb38d7cf61c0db8ee05357907d1db0ae7cd Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Mon, 4 May 2026 10:41:13 -0400
Subject: [PATCH v2_PG18 1/4] Alias WAL block references for some record types
A future commit will add block references for pages of the VM to update,
insert, delete, lock, and multi-insert WAL records. Before doing that,
alias the heap block references so that they are easy to distinguish.
Author: Melanie Plageman <melanieplageman@gmail.com>
Discussion: https://postgr.es/m/66mqpfyti3qhfttcsv6r2lbvqqd32rrmpn6i47ovrsnvguts46%40gou54xc>
Backpatch through: 17
---
src/backend/access/heap/heapam.c | 43 +++++++++++---------
src/backend/access/heap/heapam_xlog.c | 56 +++++++++++++++++----------
src/include/access/heapam_xlog.h | 32 +++++++++++----
3 files changed, 86 insertions(+), 45 deletions(-)
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 6203e3d7f8d..3c3b65b3cbf 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2218,10 +2218,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
* write the whole page to the xlog, we don't need to store
* xl_heap_header in the xlog.
*/
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
- XLogRegisterBufData(0, &xlhdr, SizeOfHeapHeader);
+ XLogRegisterBuffer(HEAP_INSERT_BLKREF_HEAP, buffer,
+ REGBUF_STANDARD | bufflags);
+ XLogRegisterBufData(HEAP_INSERT_BLKREF_HEAP, &xlhdr,
+ SizeOfHeapHeader);
/* PG73FORMAT: write bitmap [+ padding] [+ oid] + data */
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_INSERT_BLKREF_HEAP,
(char *) heaptup->t_data + SizeofHeapTupleHeader,
heaptup->t_len - SizeofHeapTupleHeader);
@@ -2619,9 +2621,11 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogBeginInsert();
XLogRegisterData(xlrec, tupledata - scratch.data);
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
+ XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_HEAP, buffer,
+ REGBUF_STANDARD | bufflags);
- XLogRegisterBufData(0, tupledata, totaldatalen);
+ XLogRegisterBufData(HEAP_MULTI_INSERT_BLKREF_HEAP, tupledata,
+ totaldatalen);
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
@@ -3112,7 +3116,7 @@ l1:
XLogBeginInsert();
XLogRegisterData(&xlrec, SizeOfHeapDelete);
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_DELETE_BLKREF_HEAP, buffer, REGBUF_STANDARD);
/*
* Log replica identity of the deleted tuple if there is one
@@ -3883,7 +3887,7 @@ l2:
XLogRecPtr recptr;
XLogBeginInsert();
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buffer, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&oldtup.t_self);
xlrec.xmax = xmax_lock_old_tuple;
@@ -5219,7 +5223,7 @@ failed:
XLogRecPtr recptr;
XLogBeginInsert();
- XLogRegisterBuffer(0, *buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, *buffer, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&tuple->t_self);
xlrec.xmax = xid;
@@ -5971,7 +5975,7 @@ l4:
Page page = BufferGetPage(buf);
XLogBeginInsert();
- XLogRegisterBuffer(0, buf, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buf, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&mytup.t_self);
xlrec.xmax = new_xmax;
@@ -8972,9 +8976,9 @@ log_heap_update(Relation reln, Buffer oldbuf,
if (need_tuple_data)
bufflags |= REGBUF_KEEP_DATA;
- XLogRegisterBuffer(0, newbuf, bufflags);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_NEW, newbuf, bufflags);
if (oldbuf != newbuf)
- XLogRegisterBuffer(1, oldbuf, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_OLD, oldbuf, REGBUF_STANDARD);
XLogRegisterData(&xlrec, SizeOfHeapUpdate);
@@ -8987,15 +8991,18 @@ log_heap_update(Relation reln, Buffer oldbuf,
{
prefix_suffix[0] = prefixlen;
prefix_suffix[1] = suffixlen;
- XLogRegisterBufData(0, &prefix_suffix, sizeof(uint16) * 2);
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW, &prefix_suffix,
+ sizeof(uint16) * 2);
}
else if (prefixlen > 0)
{
- XLogRegisterBufData(0, &prefixlen, sizeof(uint16));
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW, &prefixlen,
+ sizeof(uint16));
}
else
{
- XLogRegisterBufData(0, &suffixlen, sizeof(uint16));
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW, &suffixlen,
+ sizeof(uint16));
}
}
@@ -9009,10 +9016,10 @@ log_heap_update(Relation reln, Buffer oldbuf,
*
* The 'data' doesn't include the common prefix or suffix.
*/
- XLogRegisterBufData(0, &xlhdr, SizeOfHeapHeader);
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW, &xlhdr, SizeOfHeapHeader);
if (prefixlen == 0)
{
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW,
(char *) newtup->t_data + SizeofHeapTupleHeader,
newtup->t_len - SizeofHeapTupleHeader - suffixlen);
}
@@ -9025,13 +9032,13 @@ log_heap_update(Relation reln, Buffer oldbuf,
/* bitmap [+ padding] [+ oid] */
if (newtup->t_data->t_hoff - SizeofHeapTupleHeader > 0)
{
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW,
(char *) newtup->t_data + SizeofHeapTupleHeader,
newtup->t_data->t_hoff - SizeofHeapTupleHeader);
}
/* data after common prefix */
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW,
(char *) newtup->t_data + newtup->t_data->t_hoff + prefixlen,
newtup->t_len - newtup->t_data->t_hoff - prefixlen - suffixlen);
}
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index eb4bd3d6ae3..6cc3bc991c3 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -350,7 +350,8 @@ heap_xlog_delete(XLogReaderState *record)
RelFileLocator target_locator;
ItemPointerData target_tid;
- XLogRecGetBlockTag(record, 0, &target_locator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_DELETE_BLKREF_HEAP, &target_locator, NULL,
+ &blkno);
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
@@ -369,7 +370,8 @@ heap_xlog_delete(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_DELETE_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = BufferGetPage(buffer);
@@ -434,7 +436,8 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerData target_tid;
XLogRedoAction action;
- XLogRecGetBlockTag(record, 0, &target_locator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_INSERT_BLKREF_HEAP, &target_locator, NULL,
+ &blkno);
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
@@ -462,13 +465,14 @@ heap_xlog_insert(XLogReaderState *record)
*/
if (XLogRecGetInfo(record) & XLOG_HEAP_INIT_PAGE)
{
- buffer = XLogInitBufferForRedo(record, 0);
+ buffer = XLogInitBufferForRedo(record, HEAP_INSERT_BLKREF_HEAP);
page = BufferGetPage(buffer);
PageInit(page, BufferGetPageSize(buffer), 0);
action = BLK_NEEDS_REDO;
}
else
- action = XLogReadBufferForRedo(record, 0, &buffer);
+ action = XLogReadBufferForRedo(record, HEAP_INSERT_BLKREF_HEAP,
+ &buffer);
if (action == BLK_NEEDS_REDO)
{
Size datalen;
@@ -479,7 +483,7 @@ heap_xlog_insert(XLogReaderState *record)
if (PageGetMaxOffsetNumber(page) + 1 < xlrec->offnum)
elog(PANIC, "invalid max offset number");
- data = XLogRecGetBlockData(record, 0, &datalen);
+ data = XLogRecGetBlockData(record, HEAP_INSERT_BLKREF_HEAP, &datalen);
newlen = datalen - SizeOfHeapHeader;
Assert(datalen > SizeOfHeapHeader && newlen <= MaxHeapTupleSize);
@@ -559,7 +563,8 @@ heap_xlog_multi_insert(XLogReaderState *record)
*/
xlrec = (xl_heap_multi_insert *) XLogRecGetData(record);
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_MULTI_INSERT_BLKREF_HEAP, &rlocator, NULL,
+ &blkno);
/* check that the mutually exclusive flags are not both set */
Assert(!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
@@ -582,13 +587,14 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (isinit)
{
- buffer = XLogInitBufferForRedo(record, 0);
+ buffer = XLogInitBufferForRedo(record, HEAP_MULTI_INSERT_BLKREF_HEAP);
page = BufferGetPage(buffer);
PageInit(page, BufferGetPageSize(buffer), 0);
action = BLK_NEEDS_REDO;
}
else
- action = XLogReadBufferForRedo(record, 0, &buffer);
+ action = XLogReadBufferForRedo(record, HEAP_MULTI_INSERT_BLKREF_HEAP,
+ &buffer);
if (action == BLK_NEEDS_REDO)
{
char *tupdata;
@@ -596,7 +602,8 @@ heap_xlog_multi_insert(XLogReaderState *record)
Size len;
/* Tuples are stored as block data */
- tupdata = XLogRecGetBlockData(record, 0, &len);
+ tupdata = XLogRecGetBlockData(record, HEAP_MULTI_INSERT_BLKREF_HEAP,
+ &len);
endptr = tupdata + len;
page = (Page) BufferGetPage(buffer);
@@ -713,8 +720,10 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
oldtup.t_data = NULL;
oldtup.t_len = 0;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &newblk);
- if (XLogRecGetBlockTagExtended(record, 1, NULL, NULL, &oldblk, NULL))
+ XLogRecGetBlockTag(record, HEAP_UPDATE_BLKREF_NEW, &rlocator, NULL,
+ &newblk);
+ if (XLogRecGetBlockTagExtended(record, HEAP_UPDATE_BLKREF_OLD, NULL, NULL,
+ &oldblk, NULL))
{
/* HOT updates are never done across pages */
Assert(!hot_update);
@@ -750,7 +759,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
*/
/* Deal with old tuple version */
- oldaction = XLogReadBufferForRedo(record, (oldblk == newblk) ? 0 : 1,
+ oldaction = XLogReadBufferForRedo(record, (oldblk == newblk) ?
+ HEAP_UPDATE_BLKREF_NEW : HEAP_UPDATE_BLKREF_OLD,
&obuffer);
if (oldaction == BLK_NEEDS_REDO)
{
@@ -800,13 +810,14 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
}
else if (XLogRecGetInfo(record) & XLOG_HEAP_INIT_PAGE)
{
- nbuffer = XLogInitBufferForRedo(record, 0);
+ nbuffer = XLogInitBufferForRedo(record, HEAP_UPDATE_BLKREF_NEW);
page = (Page) BufferGetPage(nbuffer);
PageInit(page, BufferGetPageSize(nbuffer), 0);
newaction = BLK_NEEDS_REDO;
}
else
- newaction = XLogReadBufferForRedo(record, 0, &nbuffer);
+ newaction = XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_NEW,
+ &nbuffer);
/*
* The visibility map may need to be fixed even if the heap page is
@@ -831,7 +842,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Size datalen;
Size tuplen;
- recdata = XLogRecGetBlockData(record, 0, &datalen);
+ recdata = XLogRecGetBlockData(record, HEAP_UPDATE_BLKREF_NEW,
+ &datalen);
recdata_end = recdata + datalen;
page = BufferGetPage(nbuffer);
@@ -1015,7 +1027,8 @@ heap_xlog_lock(XLogReaderState *record)
BlockNumber block;
Relation reln;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
+ XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
+ &block);
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
@@ -1025,7 +1038,8 @@ heap_xlog_lock(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = (Page) BufferGetPage(buffer);
@@ -1091,7 +1105,8 @@ heap_xlog_lock_updated(XLogReaderState *record)
BlockNumber block;
Relation reln;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
+ XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
+ &block);
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
@@ -1101,7 +1116,8 @@ heap_xlog_lock_updated(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = BufferGetPage(buffer);
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 277df6b3cf0..55e3c7b0015 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -110,6 +110,9 @@
(XLH_DELETE_CONTAINS_OLD_TUPLE | XLH_DELETE_CONTAINS_OLD_KEY)
/* This is what we need to know about delete */
+#define HEAP_DELETE_BLKREF_HEAP 0
+#define HEAP_DELETE_BLKREF_VM 1
+
typedef struct xl_heap_delete
{
TransactionId xmax; /* xmax of the deleted tuple */
@@ -157,12 +160,15 @@ typedef struct xl_heap_header
#define SizeOfHeapHeader (offsetof(xl_heap_header, t_hoff) + sizeof(uint8))
/* This is what we need to know about insert */
+#define HEAP_INSERT_BLKREF_HEAP 0
+#define HEAP_INSERT_BLKREF_VM 1
+
typedef struct xl_heap_insert
{
OffsetNumber offnum; /* inserted tuple's offset */
uint8 flags;
- /* xl_heap_header & TUPLE DATA in backup block 0 */
+ /* xl_heap_header & TUPLE DATA in HEAP_INSERT_BLKREF_HEAP */
} xl_heap_insert;
#define SizeOfHeapInsert (offsetof(xl_heap_insert, flags) + sizeof(uint8))
@@ -173,10 +179,14 @@ typedef struct xl_heap_insert
* The main data of the record consists of this xl_heap_multi_insert header.
* 'offsets' array is omitted if the whole page is reinitialized
* (XLOG_HEAP_INIT_PAGE).
- *
- * In block 0's data portion, there is an xl_multi_insert_tuple struct,
- * followed by the tuple data for each tuple. There is padding to align
- * each xl_multi_insert_tuple struct.
+ */
+#define HEAP_MULTI_INSERT_BLKREF_HEAP 0
+#define HEAP_MULTI_INSERT_BLKREF_VM 1
+
+/*
+ * In HEAP_MULTI_INSERT_BLKREF_HEAP's data portion, there is an
+ * xl_multi_insert_tuple struct, followed by the tuple data for each tuple.
+ * There is padding to align each xl_multi_insert_tuple struct.
*/
typedef struct xl_heap_multi_insert
{
@@ -201,7 +211,7 @@ typedef struct xl_multi_insert_tuple
/*
* This is what we need to know about update|hot_update
*
- * Backup blk 0: new page
+ * HEAP_UPDATE_BLKREF_NEW: new page
*
* If XLH_UPDATE_PREFIX_FROM_OLD or XLH_UPDATE_SUFFIX_FROM_OLD flags are set,
* the prefix and/or suffix come first, as one or two uint16s.
@@ -213,8 +223,13 @@ typedef struct xl_multi_insert_tuple
* If XLH_UPDATE_CONTAINS_NEW_TUPLE flag is given, the tuple data is
* included even if a full-page image was taken.
*
- * Backup blk 1: old page, if different. (no data, just a reference to the blk)
+ * HEAP_UPDATE_BLKREF_OLD: old page, if different. (no data, just a reference
+ * to the block)
*/
+
+#define HEAP_UPDATE_BLKREF_NEW 0
+#define HEAP_UPDATE_BLKREF_OLD 1
+
typedef struct xl_heap_update
{
TransactionId old_xmax; /* xmax of the old tuple */
@@ -393,6 +408,9 @@ typedef struct xlhp_prune_items
#define XLH_LOCK_ALL_FROZEN_CLEARED 0x01
/* This is what we need to know about lock */
+#define HEAP_LOCK_BLKREF_HEAP 0
+#define HEAP_LOCK_BLKREF_VM 1
+
typedef struct xl_heap_lock
{
TransactionId xmax; /* might be a MultiXactId */
--
2.43.0
v2_PG18-0002-Fix-WAL-logging-of-VM-clears.patch (text/x-patch, 33.7K) download
From 5d16ef487aac7ab9e35d14950d8de7ba6e73351b Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 12:53:49 -0400
Subject: [PATCH v2_PG18 2/4] Fix WAL logging of VM clears
Previously, we failed to register the visibility map buffer when
emitting WAL after clearing the VM bits for heap pages (recovery code
read the VM page normally). This meant that we couldn't emit FPIs of VM
pages when clearing VM bits. While this would not result in a checksum
error because the visibility map is read with ZERO_ON_ERROR, the WAL
summarizer only includes pages which were registered, so a restore from
an incremental backup would not include the modified VM pages -- leading
to data corruption.
Fix this by registering the VM buffer in the WAL record when clearing VM
bits. This also means the VM buffer must be locked throughout the
duration of the critical section where we make changes to the VM and
heap page and emit WAL.
Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/flat/CAAKRu_bn%2Be7F4yPFBgFbnP%2BsyJRKyNK092bjD2LKvZW7O4Svag>
Backpatch-through: 17
---
contrib/pg_surgery/heap_surgery.c | 35 ++--
src/backend/access/heap/heapam.c | 259 ++++++++++++++++++++----
src/backend/access/heap/heapam_xlog.c | 237 ++++++++++++++++------
src/backend/access/heap/visibilitymap.c | 22 +-
src/bin/pg_walsummary/t/002_blocks.pl | 7 +-
src/include/access/heapam_xlog.h | 15 +-
src/include/access/visibilitymap.h | 2 +
7 files changed, 450 insertions(+), 127 deletions(-)
diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c
index 3e86283beb7..81a96023eef 100644
--- a/contrib/pg_surgery/heap_surgery.c
+++ b/contrib/pg_surgery/heap_surgery.c
@@ -17,6 +17,7 @@
#include "access/visibilitymap.h"
#include "access/xloginsert.h"
#include "catalog/pg_am_d.h"
+#include "catalog/pg_control.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
#include "utils/acl.h"
@@ -233,11 +234,14 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
}
/*
- * Before entering the critical section, pin the visibility map page
- * if it appears to be necessary.
+ * Before entering the critical section, pin and lock the visibility
+ * map page if it appears to be necessary.
*/
if (heap_force_opt == HEAP_FORCE_KILL && PageIsAllVisible(page))
+ {
visibilitymap_pin(rel, blkno, &vmbuf);
+ LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+ }
/* No ereport(ERROR) from here until all the changes are logged. */
START_CRIT_SECTION();
@@ -266,10 +270,10 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
*/
if (PageIsAllVisible(page))
{
- PageClearAllVisible(page);
- visibilitymap_clear(rel, blkno, vmbuf,
- VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear_locked(rel, blkno, vmbuf,
+ VISIBILITYMAP_VALID_BITS);
did_modify_vm = true;
+ PageClearAllVisible(page);
}
}
else
@@ -320,19 +324,26 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
/* XLOG stuff */
if (RelationNeedsWAL(rel))
- log_newpage_buffer(buf, true);
+ {
+ XLogRecPtr recptr;
+
+ XLogBeginInsert();
+ XLogRegisterBuffer(0, buf, REGBUF_STANDARD | REGBUF_FORCE_IMAGE);
+ if (did_modify_vm)
+ XLogRegisterBuffer(1, vmbuf, REGBUF_FORCE_IMAGE);
+ recptr = XLogInsert(RM_XLOG_ID, XLOG_FPI);
+ if (did_modify_vm)
+ PageSetLSN(BufferGetPage(vmbuf), recptr);
+ PageSetLSN(BufferGetPage(buf), recptr);
+ }
}
- /* WAL log the VM page if it was modified. */
- if (did_modify_vm && RelationNeedsWAL(rel))
- log_newpage_buffer(vmbuf, false);
-
END_CRIT_SECTION();
UnlockReleaseBuffer(buf);
- if (vmbuf != InvalidBuffer)
- ReleaseBuffer(vmbuf);
+ if (BufferIsValid(vmbuf))
+ UnlockReleaseBuffer(vmbuf);
/* Update the current_start_ptr before moving to the next page. */
curr_start_ptr = next_start_ptr;
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 3c3b65b3cbf..344e1f82bd7 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -58,7 +58,8 @@
static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup,
TransactionId xid, CommandId cid, int options);
static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup,
+ Buffer vmbuffer_old, Buffer newbuf,
+ Buffer vmbuffer_new, HeapTuple oldtup,
HeapTuple newtup, HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared);
#ifdef USE_ASSERT_CHECKING
@@ -2135,10 +2136,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
if (PageIsAllVisible(BufferGetPage(buffer)))
{
all_visible_cleared = true;
+
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear_locked(relation,
+ ItemPointerGetBlockNumber(&(heaptup->t_self)),
+ vmbuffer, VISIBILITYMAP_VALID_BITS);
PageClearAllVisible(BufferGetPage(buffer));
- visibilitymap_clear(relation,
- ItemPointerGetBlockNumber(&(heaptup->t_self)),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
}
/*
@@ -2230,13 +2233,22 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (all_visible_cleared)
+ XLogRegisterBuffer(HEAP_INSERT_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, info);
PageSetLSN(page, recptr);
+
+ if (all_visible_cleared)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
UnlockReleaseBuffer(buffer);
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
@@ -2505,10 +2517,11 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
{
all_visible_cleared = true;
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear_locked(relation,
+ BufferGetBlockNumber(buffer),
+ vmbuffer, VISIBILITYMAP_VALID_BITS);
PageClearAllVisible(page);
- visibilitymap_clear(relation,
- BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
}
else if (all_frozen_set)
PageSetAllVisible(page);
@@ -2623,6 +2636,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogRegisterData(xlrec, tupledata - scratch.data);
XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_HEAP, buffer,
REGBUF_STANDARD | bufflags);
+ if (all_visible_cleared)
+ XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_VM, vmbuffer, 0);
XLogRegisterBufData(HEAP_MULTI_INSERT_BLKREF_HEAP, tupledata,
totaldatalen);
@@ -2633,10 +2648,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
recptr = XLogInsert(RM_HEAP2_ID, info);
PageSetLSN(page, recptr);
+ if (all_visible_cleared)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
/*
* If we've frozen everything on the page, update the visibilitymap.
* We're already holding pin on the vmbuffer.
@@ -3054,9 +3074,10 @@ l1:
if (PageIsAllVisible(page))
{
all_visible_cleared = true;
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear_locked(relation, BufferGetBlockNumber(buffer),
+ vmbuffer, VISIBILITYMAP_VALID_BITS);
PageClearAllVisible(page);
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
}
/* store transaction information of xact deleting the tuple */
@@ -3137,13 +3158,23 @@ l1:
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (all_visible_cleared)
+ XLogRegisterBuffer(HEAP_DELETE_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE);
PageSetLSN(page, recptr);
+
+ if (all_visible_cleared)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
if (vmbuffer != InvalidBuffer)
@@ -3261,7 +3292,8 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
HeapTuple heaptup;
HeapTuple old_key_tuple = NULL;
bool old_key_copied = false;
- Page page;
+ Page page,
+ newpage;
BlockNumber block;
MultiXactStatus mxact_status;
Buffer buffer,
@@ -3278,6 +3310,8 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
bool key_intact;
bool all_visible_cleared = false;
bool all_visible_cleared_new = false;
+ bool modified_vmbuffer = false;
+ bool modified_vmbuffer_new = false;
bool checked_lockers;
bool locker_remains;
bool id_has_external = false;
@@ -3386,8 +3420,11 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
UnlockReleaseBuffer(buffer);
Assert(!have_tuple_lock);
- if (vmbuffer != InvalidBuffer)
+ if (BufferIsValid(vmbuffer))
+ {
ReleaseBuffer(vmbuffer);
+ vmbuffer = InvalidBuffer;
+ }
tmfd->ctid = *otid;
tmfd->xmax = InvalidTransactionId;
tmfd->cmax = InvalidCommandId;
@@ -3690,8 +3727,11 @@ l2:
UnlockReleaseBuffer(buffer);
if (have_tuple_lock)
UnlockTupleTuplock(relation, &(oldtup.t_self), *lockmode);
- if (vmbuffer != InvalidBuffer)
+ if (BufferIsValid(vmbuffer))
+ {
ReleaseBuffer(vmbuffer);
+ vmbuffer = InvalidBuffer;
+ }
*update_indexes = TU_None;
bms_free(hot_attrs);
@@ -3874,10 +3914,15 @@ l2:
* overhead would be unchanged, that doesn't seem necessarily
* worthwhile.
*/
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear_locked(relation, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ else
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ }
MarkBufferDirty(buffer);
@@ -3896,12 +3941,23 @@ l2:
xlrec.flags =
cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (cleared_all_frozen)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
/*
@@ -3987,6 +4043,8 @@ l2:
heaptup = newtup;
}
+ newpage = BufferGetPage(newbuf);
+
/*
* We're about to do the actual update -- check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -4100,20 +4158,85 @@ l2:
/* record address of new tuple in t_ctid of old one */
oldtup.t_data->t_ctid = heaptup->t_self;
- /* clear PD_ALL_VISIBLE flags, reset all visibilitymap bits */
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * Clear PD_ALL_VISIBLE flags and reset visibility map bits for any heap
+ * pages that were all-visible. If there are two heap pages, we may need
+ * to clear VM bits for both.
+ */
+ if (PageIsAllVisible(page) &&
+ newbuf != buffer && PageIsAllVisible(newpage) &&
+ vmbuffer_new == vmbuffer)
{
+ /*
+ * This is the more complicated case: both the new and old heap pages
+ * are all-visible and both their VM bits are on the same page of the
+ * VM, so we register a single VM buffer as HEAP_UPDATE_BLKREF_VM_NEW
+ * in the WAL record. We must be careful to only lock and register one
+ * buffer, even though we modify it twice -- once for each heap
+ * block's VM bits.
+ */
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear_locked(relation, block,
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear_locked(relation, BufferGetBlockNumber(newbuf),
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS);
+ PageClearAllVisible(page);
+ PageClearAllVisible(newpage);
+ modified_vmbuffer_new = true;
all_visible_cleared = true;
- PageClearAllVisible(BufferGetPage(buffer));
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ all_visible_cleared_new = true;
}
- if (newbuf != buffer && PageIsAllVisible(BufferGetPage(newbuf)))
+ else
{
- all_visible_cleared_new = true;
- PageClearAllVisible(BufferGetPage(newbuf));
- visibilitymap_clear(relation, BufferGetBlockNumber(newbuf),
- vmbuffer_new, VISIBILITYMAP_VALID_BITS);
+ /*
+ * In all the remaining cases, we will clear at most one heap block's
+ * VM bits per VM page.
+ */
+ bool old_all_visible = PageIsAllVisible(page);
+ bool new_all_visible = newbuf != buffer && PageIsAllVisible(newpage);
+ Buffer vmbuffers[2] = {
+ old_all_visible ? vmbuffer : InvalidBuffer,
+ new_all_visible ? vmbuffer_new : InvalidBuffer
+ };
+
+ /*
+ * When both pages need different VM pages cleared, acquire the VM
+ * buffer locks in VM block order to avoid deadlocks between backends
+ * updating tuples in opposite directions across VM pages.
+ */
+ if (old_all_visible && new_all_visible &&
+ BufferGetBlockNumber(vmbuffers[0]) > BufferGetBlockNumber(vmbuffers[1]))
+ {
+ Buffer t = vmbuffers[0];
+
+ vmbuffers[0] = vmbuffers[1];
+ vmbuffers[1] = t;
+ }
+
+ Assert((!BufferIsValid(vmbuffers[0]) && !BufferIsValid(vmbuffers[1])) ||
+ vmbuffers[0] != vmbuffers[1]);
+
+ if (BufferIsValid(vmbuffers[0]))
+ LockBuffer(vmbuffers[0], BUFFER_LOCK_EXCLUSIVE);
+ if (BufferIsValid(vmbuffers[1]))
+ LockBuffer(vmbuffers[1], BUFFER_LOCK_EXCLUSIVE);
+
+ if (old_all_visible)
+ {
+ visibilitymap_clear_locked(relation, block,
+ vmbuffer, VISIBILITYMAP_VALID_BITS);
+ modified_vmbuffer = true;
+ PageClearAllVisible(page);
+ all_visible_cleared = true;
+ }
+ if (new_all_visible)
+ {
+ visibilitymap_clear_locked(relation, BufferGetBlockNumber(newbuf),
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS);
+ modified_vmbuffer_new = true;
+ PageClearAllVisible(newpage);
+ all_visible_cleared_new = true;
+ }
}
if (newbuf != buffer)
@@ -4136,19 +4259,30 @@ l2:
}
recptr = log_heap_update(relation, buffer,
- newbuf, &oldtup, heaptup,
+ modified_vmbuffer ? vmbuffer : InvalidBuffer,
+ newbuf,
+ modified_vmbuffer_new ? vmbuffer_new : InvalidBuffer,
+ &oldtup, heaptup,
old_key_tuple,
all_visible_cleared,
all_visible_cleared_new);
if (newbuf != buffer)
- {
- PageSetLSN(BufferGetPage(newbuf), recptr);
- }
+ PageSetLSN(newpage, recptr);
PageSetLSN(BufferGetPage(buffer), recptr);
+
+ if (modified_vmbuffer)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
+ if (modified_vmbuffer_new)
+ PageSetLSN(BufferGetPage(vmbuffer_new), recptr);
}
END_CRIT_SECTION();
+ if (modified_vmbuffer)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ if (modified_vmbuffer_new)
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_UNLOCK);
+
if (newbuf != buffer)
LockBuffer(newbuf, BUFFER_LOCK_UNLOCK);
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
@@ -5197,10 +5331,15 @@ failed:
tuple->t_data->t_ctid = *tid;
/* Clear only the all-frozen bit on visibility map if needed */
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear_locked(relation, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ else
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ }
MarkBufferDirty(*buffer);
@@ -5232,15 +5371,25 @@ failed:
xlrec.flags = cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
/* we don't decode row locks atm, so no need to log the origin */
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (cleared_all_frozen)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
result = TM_Ok;
out_locked:
@@ -5951,10 +6100,15 @@ l4:
xid, mode, false,
&new_xmax, &new_infomask, &new_infomask2);
- if (PageIsAllVisible(BufferGetPage(buf)) &&
- visibilitymap_clear(rel, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(BufferGetPage(buf)))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear_locked(rel, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ else
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ }
START_CRIT_SECTION();
@@ -5985,13 +6139,23 @@ l4:
XLogRegisterData(&xlrec, SizeOfHeapLockUpdated);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_LOCK_UPDATED);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (cleared_all_frozen)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
next:
/* if we find the end of update chain, we're done. */
if (mytup.t_data->t_infomask & HEAP_XMAX_INVALID ||
@@ -8848,8 +9012,9 @@ log_heap_visible(Relation rel, Buffer heap_buffer, Buffer vm_buffer,
* have modified the buffer(s) and marked them dirty.
*/
static XLogRecPtr
-log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup, HeapTuple newtup,
+log_heap_update(Relation reln, Buffer oldbuf, Buffer vmbuffer_old,
+ Buffer newbuf, Buffer vmbuffer_new,
+ HeapTuple oldtup, HeapTuple newtup,
HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared)
{
@@ -9058,6 +9223,20 @@ log_heap_update(Relation reln, Buffer oldbuf,
old_key_tuple->t_len - SizeofHeapTupleHeader);
}
+ /*
+ * Register VM buffers. If the old and new heap pages' VM bits are on the
+ * same VM page, the caller passes only vmbuffer_new (mirroring the heap
+ * page convention where block 0 = new is always registered).
+ */
+ Assert((BufferIsInvalid(vmbuffer_old) && BufferIsInvalid(vmbuffer_new)) ||
+ (vmbuffer_old != vmbuffer_new));
+
+ if (BufferIsValid(vmbuffer_new))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_NEW, vmbuffer_new, 0);
+
+ if (BufferIsValid(vmbuffer_old))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_OLD, vmbuffer_old, 0);
+
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index 6cc3bc991c3..4ad56d76768 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -22,6 +22,51 @@
#include "storage/freespace.h"
#include "storage/standby.h"
+/*
+ * Helper to clear visibility map bits during heap redo.
+ *
+ * This handles records that modify one heap block and its corresponding VM
+ * block. More complex cases, such as update records that can touch multiple
+ * heap or VM blocks, must handle VM replay directly. 'flags' specifies which
+ * visibility map bits to clear.
+ */
+static void
+heap_xlog_vm_clear(XLogReaderState *record,
+ XLogRecPtr lsn, RelFileLocator target_locator,
+ BlockNumber heap_blkno,
+ int vm_block_id, uint8 flags)
+{
+ Relation reln = CreateFakeRelcacheEntry(target_locator);
+ Buffer vmbuffer = InvalidBuffer;
+
+ /*
+ * If this record includes the VM block, use it for redo. Older minor
+ * releases did not register the VM buffer when clearing the VM, so keep
+ * the fallback path to support replay of WAL generated before that fix.
+ */
+ if (XLogRecHasBlockRef(record, vm_block_id))
+ {
+ if (XLogReadBufferForRedo(record, vm_block_id,
+ &vmbuffer) == BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln,
+ heap_blkno, vmbuffer,
+ flags);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else
+ {
+ visibilitymap_pin(reln, heap_blkno, &vmbuffer);
+ visibilitymap_clear(reln, heap_blkno, vmbuffer, flags);
+ ReleaseBuffer(vmbuffer);
+ }
+
+ FreeFakeRelcacheEntry(reln);
+}
+
/*
* Replay XLOG_HEAP2_PRUNE_* records.
@@ -360,15 +405,9 @@ heap_xlog_delete(XLogReaderState *record)
* already up-to-date.
*/
if (xlrec->flags & XLH_DELETE_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(target_locator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
+ heap_xlog_vm_clear(record, lsn, target_locator,
+ blkno, HEAP_DELETE_BLKREF_VM,
+ VISIBILITYMAP_VALID_BITS);
if (XLogReadBufferForRedo(record, HEAP_DELETE_BLKREF_HEAP,
&buffer) == BLK_NEEDS_REDO)
@@ -449,15 +488,9 @@ heap_xlog_insert(XLogReaderState *record)
* already up-to-date.
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(target_locator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
+ heap_xlog_vm_clear(record, lsn, target_locator,
+ blkno, HEAP_INSERT_BLKREF_VM,
+ VISIBILITYMAP_VALID_BITS);
/*
* If we inserted the first and only tuple on the page, re-initialize the
@@ -571,19 +604,19 @@ heap_xlog_multi_insert(XLogReaderState *record)
(xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
/*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
+ * If required, clear the VM first, to prevent all-visible temporarily
+ * being set for a heap page that's not all visible anymore.
+ *
+ * If it were possible for XLH_INSERT_ALL_VISIBLE_CLEARED and
+ * XLH_INSERT_ALL_FROZEN_SET to be present in the same record, doing the
+ * XLogReadBufferForRedo() before the PageSetAllVisible() below would be a
+ * problem, as it'd violate the rule that a heap page must never be set
+ * all-visible in the VM while its PD_ALL_VISIBLE is clear.
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
+ heap_xlog_vm_clear(record, lsn, rlocator,
+ blkno, HEAP_MULTI_INSERT_BLKREF_VM,
+ VISIBILITYMAP_VALID_BITS);
if (isinit)
{
@@ -698,6 +731,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Buffer obuffer,
nbuffer;
Page page;
+ bool new_cleared,
+ old_cleared;
OffsetNumber offnum;
ItemId lp = NULL;
HeapTupleData oldtup;
@@ -737,14 +772,107 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
*/
- if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED)
+ new_cleared = (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0;
+ old_cleared = (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) != 0;
+ if (new_cleared || old_cleared)
{
Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
+ bool has_vm_old = XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_OLD);
+ bool has_vm_new = XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_NEW);
+
+ if (has_vm_new || has_vm_old)
+ {
+ /*
+ * If both the old and new heap pages were all-visible and their
+ * VM bits are on the same VM page, that single VM page is
+ * registered as HEAP_UPDATE_BLKREF_VM_NEW. Clear both heap
+ * blocks' VM bits from the single provided VM buffer.
+ */
+ bool same_vm_page = new_cleared && old_cleared &&
+ !has_vm_old && has_vm_new;
+
+ if (same_vm_page)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW, &vmbuffer) ==
+ BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear_locked(reln, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else
+ {
+ /*
+ * We can be sure that we need to clear at most one heap
+ * page's VM bits in each registered VM buffer.
+ */
+ if (new_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW, &vmbuffer) ==
+ BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ if (old_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_OLD, &vmbuffer) ==
+ BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ }
+ }
+ else
+ {
+ /*
+ * Backwards compatibility path. Previously, the VM buffers were
+ * not registered in the WAL record. We need this path to replay
+ * WAL generated by a not-yet-patched primary during upgrade.
+ */
+ if (old_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ visibilitymap_pin(reln, oldblk, &vmbuffer);
+ visibilitymap_clear(reln, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
+ if (new_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ visibilitymap_pin(reln, newblk, &vmbuffer);
+ visibilitymap_clear(reln, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
+ }
- visibilitymap_pin(reln, oldblk, &vmbuffer);
- visibilitymap_clear(reln, oldblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -819,21 +947,6 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
newaction = XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_NEW,
&nbuffer);
- /*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
- */
- if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, newblk, &vmbuffer);
- visibilitymap_clear(reln, newblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
-
/* Deal with new tuple */
if (newaction == BLK_NEEDS_REDO)
{
@@ -1022,20 +1135,15 @@ heap_xlog_lock(XLogReaderState *record)
*/
if (xlrec->flags & XLH_LOCK_ALL_FROZEN_CLEARED)
{
- RelFileLocator rlocator;
- Buffer vmbuffer = InvalidBuffer;
BlockNumber block;
- Relation reln;
+ RelFileLocator rlocator;
XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
&block);
- reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ heap_xlog_vm_clear(record, lsn, rlocator,
+ block, HEAP_LOCK_BLKREF_VM,
+ VISIBILITYMAP_ALL_FROZEN);
}
if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
@@ -1100,20 +1208,15 @@ heap_xlog_lock_updated(XLogReaderState *record)
*/
if (xlrec->flags & XLH_LOCK_ALL_FROZEN_CLEARED)
{
- RelFileLocator rlocator;
- Buffer vmbuffer = InvalidBuffer;
BlockNumber block;
- Relation reln;
+ RelFileLocator rlocator;
XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
&block);
- reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ heap_xlog_vm_clear(record, lsn, rlocator,
+ block, HEAP_LOCK_BLKREF_VM,
+ VISIBILITYMAP_ALL_FROZEN);
}
if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 1874a3fda37..89079b52a2f 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -135,9 +135,27 @@ static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks);
* You must pass a buffer containing the correct map page to this function.
* Call visibilitymap_pin first to pin the right one. This function doesn't do
* any I/O. Returns true if any bits have been cleared and false otherwise.
+ *
+ * This is retained for backwards compatability, as it is usually necessary to
+ * register the VM buffer in the WAL record and this necessitates holding the
+ * lock for longer than it is held here.
*/
bool
visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
+{
+ bool cleared = false;
+
+ LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+
+ cleared = visibilitymap_clear_locked(rel, heapBlk, vmbuf, flags);
+
+ LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
+
+ return cleared;
+}
+
+bool
+visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
{
BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
@@ -157,7 +175,8 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
elog(ERROR, "wrong buffer passed to visibilitymap_clear");
- LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+ Assert(BufferIsExclusiveLocked(vmbuf));
+
map = PageGetContents(BufferGetPage(vmbuf));
if (map[mapByte] & mask)
@@ -168,7 +187,6 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
cleared = true;
}
- LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
return cleared;
}
diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl
index 0f98c7df82e..fcb701e1b98 100644
--- a/src/bin/pg_walsummary/t/002_blocks.pl
+++ b/src/bin/pg_walsummary/t/002_blocks.pl
@@ -93,13 +93,14 @@ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
split(m@/@, $end_lsn);
ok(-f $filename, "WAL summary file exists");
-# Run pg_walsummary on it. We expect exactly two blocks to be modified,
-# block 0 and one other.
+# Run pg_walsummary on it. We expect exactly three blocks to be modified,
+# block 0 (old tuple), another block (new tuple), and the block for the VM.
my ($stdout, $stderr) = run_command([ 'pg_walsummary', '-i', $filename ]);
note($stdout);
@lines = split(/\n/, $stdout);
like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified");
+like($stdout, qr/FORK vm: block 0$/m, "stdout shows VM block 0 modified");
is($stderr, '', 'stderr is empty');
-is(0 + @lines, 2, "UPDATE modified 2 blocks");
+is(0 + @lines, 3, "UPDATE modified 3 blocks");
done_testing();
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 55e3c7b0015..752f69a2691 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -225,10 +225,19 @@ typedef struct xl_multi_insert_tuple
*
* HEAP_UPDATE_BLKREF_OLD: old page, if different. (no data, just a reference
* to the block)
+ *
+ * HEAP_UPDATE_BLKREF_VM_NEW: VM page covering the new heap page. Registered
+ * whenever XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED is set. Also covers the old heap
+ * page's VM bits when both heap pages map to the same VM page.
+ *
+ * HEAP_UPDATE_BLKREF_VM_OLD: VM page covering the old heap page. Only
+ * registered when XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED is set and the old heap
+ * page's VM bits are on a different VM page from the new heap page's.
*/
-
-#define HEAP_UPDATE_BLKREF_NEW 0
-#define HEAP_UPDATE_BLKREF_OLD 1
+#define HEAP_UPDATE_BLKREF_NEW 0
+#define HEAP_UPDATE_BLKREF_OLD 1
+#define HEAP_UPDATE_BLKREF_VM_NEW 2
+#define HEAP_UPDATE_BLKREF_VM_OLD 3
typedef struct xl_heap_update
{
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index ea889bf9ec7..1bc59c9ac33 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -26,6 +26,8 @@
#define VM_ALL_FROZEN(r, b, v) \
((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0)
+extern bool visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk,
+ Buffer vmbuf, uint8 flags);
extern bool visibilitymap_clear(Relation rel, BlockNumber heapBlk,
Buffer vmbuf, uint8 flags);
extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk,
--
2.43.0
v2_PG18-0003-Put-pg_combinebackup-debug-test-output-in-te.patch (text/x-patch, 4.2K) download
From 152e74908bb0699d31368e605e4ffed6f312d449 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 30 Apr 2026 16:06:06 -0400
Subject: [PATCH v2_PG18 3/4] Put pg_combinebackup debug test output in temp
file
Having --debug as the default option to pg_combinebackup in
init_from_backup() makes the logs extremely verbose. It is used for
debugging failures, so put the output in a temp file and only keep it
around if tests fail.
Author: Melanie Plageman <melanieplageman@gmail.com>
Disussion: https://postgr.es/m/y73coty2smkxgvelh4e32kqrt5pyn2pb7cz7x4mrz6yaxitqd5%40nzhkeazqf4bn
Backpatch-through: 17
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 83 +++++++++++++++++++++++-
1 file changed, 81 insertions(+), 2 deletions(-)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 2e31c23a6ac..3388e45b726 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -125,6 +125,8 @@ our $min_compat = 12;
# list of file reservations made by get_free_port
my @port_reservation_files;
+my @combinebackup_debug_logs;
+my %combinebackup_debug_logs_printed;
# We want to choose a server port above the range that servers typically use
# on Unix systems and below the range those systems typically use for ephemeral
@@ -898,7 +900,9 @@ pathnames that should be used for the target cluster.
To restore from an incremental backup, pass the parameter combine_with_prior
as a reference to an array of prior backup names with which this backup
-is to be combined using pg_combinebackup.
+is to be combined using pg_combinebackup. pg_combinebackup is run with
+--debug, but the output is captured in a temporary file and only copied to the
+test log if pg_combinebackup fails or if a later test fails.
Streaming replication can be enabled on this node by passing the keyword
parameter has_streaming => 1. This is disabled by default.
@@ -964,7 +968,7 @@ sub init_from_backup
}
push @combineargs, @prior_backup_path, $backup_path,
'--output' => $data_path;
- PostgreSQL::Test::Utils::system_or_bail(@combineargs);
+ _run_pg_combinebackup_with_debug_log(@combineargs);
}
elsif (defined $params{tar_program})
{
@@ -1079,6 +1083,81 @@ port = $port
return;
}
+sub _run_pg_combinebackup_with_debug_log
+{
+ my (@cmd) = @_;
+ my $debug_log = PostgreSQL::Test::Utils::tempdir('pg_combinebackup') .
+ '/debug.log';
+
+ push @combinebackup_debug_logs, $debug_log;
+
+ print("# Running: " . join(" ", @cmd) . "\n");
+ # If this pg_combinebackup invocation itself fails, print its captured
+ # output immediately. Successful invocations may still be printed later if
+ # some subsequent TAP assertion in this test script fails.
+ if (!IPC::Run::run(\@cmd, '&>' => $debug_log))
+ {
+ _print_pg_combinebackup_debug_log($debug_log);
+
+ if ($? == -1)
+ {
+ BAIL_OUT(
+ sprintf(
+ "failed to execute command \"%s\": $!", join(" ", @cmd)));
+ }
+ elsif ($? & 127)
+ {
+ BAIL_OUT(
+ sprintf(
+ "command \"%s\" died with signal %d",
+ join(" ", @cmd), $? & 127));
+ }
+ else
+ {
+ BAIL_OUT(
+ sprintf(
+ "command \"%s\" exited with value %d",
+ join(" ", @cmd), $? >> 8));
+ }
+ }
+}
+
+sub _print_pg_combinebackup_debug_log
+{
+ my ($debug_log) = @_;
+
+ return unless -e $debug_log;
+ return if $combinebackup_debug_logs_printed{$debug_log}++;
+
+ diag("pg_combinebackup debug output from $debug_log:\n" .
+ PostgreSQL::Test::Utils::slurp_file($debug_log));
+}
+
+# Keep pg_combinebackup debug logs until test exit. If pg_combinebackup itself
+# fails, its log is printed at the failure site above. If pg_combinebackup
+# succeeds but a later TAP assertion fails, print the retained logs here so the
+# regress output includes the combine step that produced the restored state.
+# Remove the logs after a clean run.
+END
+{
+ # take care not to change the script's exit value
+ my $exit_code = $?;
+
+ if ($exit_code == 0 && PostgreSQL::Test::Utils::all_tests_passing())
+ {
+ unlink @combinebackup_debug_logs;
+ }
+ else
+ {
+ foreach my $debug_log (@combinebackup_debug_logs)
+ {
+ _print_pg_combinebackup_debug_log($debug_log);
+ }
+ }
+
+ $? = $exit_code;
+}
+
=pod
=item $node->rotate_logfile()
--
2.43.0
v2_PG18-0004-Test-that-VM-clear-registers-VM-buffers.patch (text/x-patch, 7.9K) download
From 19c94660f8eb7b4e01a97760df24ead07021ea6a Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 23 Apr 2026 11:13:23 -0400
Subject: [PATCH v2_PG18 4/4] Test that VM clear registers VM buffers
WAL summarizer only includes registered buffers, so you could end up
with data corruption in incremental backups if VM clears were meant to
be covered by the incremental backup.
Author: Melanie Plageman <melanieplageman@gmail.com>
Discussion: https://postgr.es/m/oqcsevg35xjan2327x5kdfth6q4fgeqboxfo3v3imeyih2uiny%406sez5dzxl6nt
Backpatch-through: 17
---
src/bin/pg_combinebackup/meson.build | 1 +
.../pg_combinebackup/t/012_vm_consistency.pl | 205 ++++++++++++++++++
2 files changed, 206 insertions(+)
create mode 100644 src/bin/pg_combinebackup/t/012_vm_consistency.pl
diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build
index bbc4c5735ba..757b78e2fe8 100644
--- a/src/bin/pg_combinebackup/meson.build
+++ b/src/bin/pg_combinebackup/meson.build
@@ -39,6 +39,7 @@ tests += {
't/009_no_full_file.pl',
't/010_hardlink.pl',
't/011_ib_truncation.pl',
+ 't/012_vm_consistency.pl',
],
}
}
diff --git a/src/bin/pg_combinebackup/t/012_vm_consistency.pl b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
new file mode 100644
index 00000000000..7b9c577d19a
--- /dev/null
+++ b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
@@ -0,0 +1,205 @@
+# Copyright (c) 2021-2026, PostgreSQL Global Development Group
+#
+# Test that heap operations clearing visibility map bits (INSERT, UPDATE,
+# DELETE, SELECT FOR UPDATE, COPY) correctly register visibility map buffers,
+# since incremental backups rely on the WAL summarizer, which only tracks
+# registered buffers.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir_short();
+my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
+
+# Set up primary with WAL summarization enabled.
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(has_archiving => 1, allows_streaming => 1);
+$primary->append_conf('postgresql.conf', <<EOF);
+summarize_wal = on
+autovacuum = off
+EOF
+$primary->start;
+
+$primary->safe_psql('postgres', q{CREATE EXTENSION pg_visibility});
+
+my @tests = (
+ {
+ label => 'INSERT',
+ table => 'vm_insert_test',
+ setup => q{CREATE TABLE vm_insert_test (id int);
+ INSERT INTO vm_insert_test DEFAULT VALUES;},
+ modify => q{INSERT INTO vm_insert_test VALUES (1)},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'DELETE',
+ table => 'vm_delete_test',
+ setup => q{CREATE TABLE vm_delete_test (id int);
+ INSERT INTO vm_delete_test VALUES (1), (2);},
+ modify => q{DELETE FROM vm_delete_test WHERE id = 1},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'UPDATE',
+ table => 'vm_update_test',
+ # Include both same-page and cross-page updates.
+ setup => q{CREATE TABLE vm_update_test (id INT, val TEXT);
+ INSERT INTO vm_update_test VALUES (1, 'same page'), (2, 'cross page');
+ INSERT INTO vm_update_test SELECT i, repeat('a', 200)
+ FROM generate_series(3, 70) i;},
+ modify => q{UPDATE vm_update_test SET id = 0 WHERE id = 1;
+ UPDATE vm_update_test SET val = repeat('b', 8000) WHERE id = 2;},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'LOCK',
+ table => 'vm_lock_test',
+ setup => q{CREATE TABLE vm_lock_test (id int);
+ INSERT INTO vm_lock_test VALUES (1), (2);},
+ modify => q{SELECT * FROM vm_lock_test WHERE id = 1 FOR UPDATE},
+ visible_op => '==',
+ frozen_op => '<',
+ },
+ {
+ label => 'COPY',
+ table => 'vm_copy_test',
+ setup => q{CREATE TABLE vm_copy_test (id int);
+ INSERT INTO vm_copy_test DEFAULT VALUES;},
+ modify => q{COPY vm_copy_test FROM PROGRAM 'echo 42'},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+);
+
+sub get_vm_summary
+{
+ my ($node, $table) = @_;
+ my $result = $node->safe_psql('postgres',
+ "SELECT all_visible, all_frozen FROM pg_visibility_map_summary('$table')");
+ my @vals = split(/\|/, $result);
+ return @vals;
+}
+
+# Confirm VACUUM (FREEZE) set VM bits before testing whether later heap
+# modifications clear those bits and are captured by incremental backup.
+sub check_vacuumed_vm
+{
+ my ($node, $test) = @_;
+ my ($all_visible, $all_frozen) = get_vm_summary($node, $test->{table});
+
+ cmp_ok($all_visible, '>', 0,
+ "$test->{label} test: pages are all-visible after vacuum");
+ cmp_ok($all_frozen, '>', 0,
+ "$test->{label} test: pages are all-frozen after vacuum");
+
+ return ($all_visible, $all_frozen);
+}
+
+# Check the VM bit counts after a heap modification against the post-vacuum
+# baseline. Most operations clear both bits; tuple locking clears all-frozen
+# without clearing all-visible.
+sub check_modified_vm
+{
+ my ($node, $test) = @_;
+ my ($post_visible, $post_frozen) = get_vm_summary($node, $test->{table});
+
+ cmp_ok($post_visible, $test->{visible_op}, $test->{pre_visible},
+ "$test->{label} test: all-visible state after modification");
+ cmp_ok($post_frozen, $test->{frozen_op}, $test->{pre_frozen},
+ "$test->{label} test: all-frozen state after modification");
+
+ return ($post_visible, $post_frozen);
+}
+
+# Verify the combined backup restored VM state exactly as it exists on the
+# primary, and ask pg_visibility to check that visible tuples are consistent.
+sub validate_restored_vm
+{
+ my ($restored, $test) = @_;
+
+ my ($primary_visible, $primary_frozen) =
+ get_vm_summary($primary, $test->{table});
+ my ($restored_visible, $restored_frozen) =
+ get_vm_summary($restored, $test->{table});
+
+ is($restored_visible, $primary_visible,
+ "$test->{label} test: restored all_visible count matches primary");
+ is($restored_frozen, $primary_frozen,
+ "$test->{label} test: restored all_frozen count matches primary");
+
+ my $corrupt_tids = $restored->safe_psql('postgres',
+ "SELECT count(*) FROM pg_check_visible('$test->{table}')");
+ is($corrupt_tids, '0',
+ "$test->{label} test: no VM corruption detected by pg_check_visible");
+}
+
+# Create and populate the tables, then vacuum freeze them to set the VM bits
+foreach my $test (@tests)
+{
+ $primary->safe_psql('postgres', $test->{setup});
+ $primary->safe_psql('postgres', "VACUUM (FREEZE) $test->{table}");
+ ($test->{pre_visible}, $test->{pre_frozen}) =
+ check_vacuumed_vm($primary, $test);
+}
+
+# Take a full backup
+my $full_name = 'full';
+my $full_path = $primary->backup_dir . "/$full_name";
+$primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $full_path,
+ '--checkpoint' => 'fast',
+ ],
+ 'full backup');
+
+# Modify the tables and check that the VM bits are as expected for that test
+# after the specified modification.
+foreach my $test (@tests)
+{
+ $primary->safe_psql('postgres', $test->{modify});
+ ($test->{post_visible}, $test->{post_frozen}) =
+ check_modified_vm($primary, $test);
+}
+
+# Take an incremental backup. This will have the changes made in the
+# modification step.
+my $incr_name = 'incr';
+my $incr_path = $primary->backup_dir . "/$incr_name";
+$primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $incr_path,
+ '--checkpoint' => 'fast',
+ '--incremental' => $full_path . '/backup_manifest',
+ ],
+ 'incremental backup');
+
+# Start a server from a combined backup composed of the incremental and full
+# backup.
+my $restored = PostgreSQL::Test::Cluster->new('restored');
+$restored->init_from_backup($primary, $incr_name,
+ combine_with_prior => [$full_name],
+ combine_mode => $mode);
+$restored->append_conf('postgresql.conf', <<EOF);
+autovacuum = off
+EOF
+$restored->start;
+$restored->safe_psql('postgres', q{CREATE EXTENSION IF NOT EXISTS pg_visibility});
+
+# Confirm that the restored server's visibility map matches the original server
+foreach my $test (@tests)
+{
+ validate_restored_vm($restored, $test);
+}
+
+$restored->stop;
+$primary->stop;
+
+done_testing();
--
2.43.0
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-05-07 12:54 Tomas Vondra <tomas@vondra.me>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 1 reply; 57+ messages in thread
From: Tomas Vondra @ 2026-05-07 12:54 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; Andres Freund <andres@anarazel.de>; +Cc: Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
On 5/6/26 23:46, Melanie Plageman wrote:
> Thanks for the review! I incorporated all your suggestions in attached
> v2 unless otherwise noted below.
>
> Based on an LLM-assisted self-review of this code, I think there is a
> pre-existing bug in heap_lock_updated_tuple_rec() where we don't reset
> cleared_all_frozen and may have a stale value if we iterate more than
> once (new tuple versions) or use the l4 goto label (retries for the
> same tuple version). This could mean we unnecessarily clear all-frozen
> on the standby (and unnecessarily register the VM buffer after my
> patch). I couldn't come up with a repro though, so I didn't include a
> patch to fix it, as I wanted to confirm that someone else also thought
> this is a bug.
>
Hmm, would this be a correctness or efficiency issue?
I'm not very familiar with this code, so I can't say if there is an
issue or not. Could you explain the sequence of steps leading to an
issue? Maybe that assumes something that's not quite possible, or it
might help constructing a reproducer.
I don't quite see how could it be related to the "l4" goto label, when
all the "goto l4" cases are before setting "cleared_all_frozen = true"
(in any given iteration).
regards
--
Tomas Vondra
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-06-23 23:55 Melanie Plageman <melanieplageman@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 1 reply; 57+ messages in thread
From: Melanie Plageman @ 2026-06-23 23:55 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; +Cc: Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
On Wed, May 6, 2026 at 5:46 PM Melanie Plageman
<melanieplageman@gmail.com> wrote:
>
> I don't see anywhere where something like this is documented. Though
> I'm not entirely sure what you mean. In the cases with one VM block
> and one heap block, you mean we should have documented that you should
> acquire both locks before making changes to either page?
>
> Thinking about this made me wonder if I should refactor the patches a
> bit to acquire the vmbuffer lock before the critical section instead.
I've done this in attached v3. I've only included proposed changes to
18, as this ended up being a pretty invasive refactor that I'd like to
get feedback on first.
While working on v3, some new tests that were added (by another
commit) exposed a bug in this patch. Because it's possible and not
incorrect for the VM bits to already be clear in the VM while
PD_ALL_VISIBLE is set on the heap page and because
visibilitymap_clear() only marks the VM buffer dirty if we actually
had to clear the bits, we can end up needing to clear PD_ALL_VISIBLE
but not dirtying the VM buffer. This basically worked before we
started registering the VM buffer in the WAL record, but now that we
do that, we have to be sure we only do so if the buffer has been
modified (because the WAL machinery asserts that the registered buffer
is dirty). Tests passed before, so we must not have had coverage of
this scenario.
Unfortunately that requirement makes an already complicated patch more
complicated -- especially heap_update(). Fixing that for heap_update()
necessitated several new variables to represent the various states.
It also creates several complicated scenarios.
For example, it is possible that though both new and old pages need
PD_ALL_VISIBLE cleared and each of their VM bits were on different
pages but the old VM buffer is omitted because it was already clear.
This was previously indistinguishable from when old and new pages VM
bits are on the same VM buffer. I think this is fixed by checking if
old page's VM bits are on vmbuffer_new...but wowza, it's all pretty
complicated now.
- Melanie
Attachments: download all (4 files, .tar.gz)
v3-0001-Alias-WAL-block-references-for-some-record-types.patch (text/x-patch, 15.1K) download
From c2ec92546047b0450e3f1adfcb03accc67ebb63a Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Mon, 4 May 2026 10:41:13 -0400
Subject: [PATCH v3 1/4] Alias WAL block references for some record types
A future commit will add block references for pages of the VM to update,
insert, delete, lock, and multi-insert WAL records. Before doing that,
alias the heap block references so that they are easy to distinguish.
Author: Melanie Plageman <melanieplageman@gmail.com>
Discussion: https://postgr.es/m/66mqpfyti3qhfttcsv6r2lbvqqd32rrmpn6i47ovrsnvguts46%40gou54xc>
Backpatch through: 17
---
src/backend/access/heap/heapam.c | 43 +++++++++++---------
src/backend/access/heap/heapam_xlog.c | 56 +++++++++++++++++----------
src/include/access/heapam_xlog.h | 32 +++++++++++----
3 files changed, 86 insertions(+), 45 deletions(-)
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 6203e3d7f8d..3c3b65b3cbf 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2218,10 +2218,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
* write the whole page to the xlog, we don't need to store
* xl_heap_header in the xlog.
*/
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
- XLogRegisterBufData(0, &xlhdr, SizeOfHeapHeader);
+ XLogRegisterBuffer(HEAP_INSERT_BLKREF_HEAP, buffer,
+ REGBUF_STANDARD | bufflags);
+ XLogRegisterBufData(HEAP_INSERT_BLKREF_HEAP, &xlhdr,
+ SizeOfHeapHeader);
/* PG73FORMAT: write bitmap [+ padding] [+ oid] + data */
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_INSERT_BLKREF_HEAP,
(char *) heaptup->t_data + SizeofHeapTupleHeader,
heaptup->t_len - SizeofHeapTupleHeader);
@@ -2619,9 +2621,11 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogBeginInsert();
XLogRegisterData(xlrec, tupledata - scratch.data);
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
+ XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_HEAP, buffer,
+ REGBUF_STANDARD | bufflags);
- XLogRegisterBufData(0, tupledata, totaldatalen);
+ XLogRegisterBufData(HEAP_MULTI_INSERT_BLKREF_HEAP, tupledata,
+ totaldatalen);
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
@@ -3112,7 +3116,7 @@ l1:
XLogBeginInsert();
XLogRegisterData(&xlrec, SizeOfHeapDelete);
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_DELETE_BLKREF_HEAP, buffer, REGBUF_STANDARD);
/*
* Log replica identity of the deleted tuple if there is one
@@ -3883,7 +3887,7 @@ l2:
XLogRecPtr recptr;
XLogBeginInsert();
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buffer, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&oldtup.t_self);
xlrec.xmax = xmax_lock_old_tuple;
@@ -5219,7 +5223,7 @@ failed:
XLogRecPtr recptr;
XLogBeginInsert();
- XLogRegisterBuffer(0, *buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, *buffer, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&tuple->t_self);
xlrec.xmax = xid;
@@ -5971,7 +5975,7 @@ l4:
Page page = BufferGetPage(buf);
XLogBeginInsert();
- XLogRegisterBuffer(0, buf, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buf, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&mytup.t_self);
xlrec.xmax = new_xmax;
@@ -8972,9 +8976,9 @@ log_heap_update(Relation reln, Buffer oldbuf,
if (need_tuple_data)
bufflags |= REGBUF_KEEP_DATA;
- XLogRegisterBuffer(0, newbuf, bufflags);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_NEW, newbuf, bufflags);
if (oldbuf != newbuf)
- XLogRegisterBuffer(1, oldbuf, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_OLD, oldbuf, REGBUF_STANDARD);
XLogRegisterData(&xlrec, SizeOfHeapUpdate);
@@ -8987,15 +8991,18 @@ log_heap_update(Relation reln, Buffer oldbuf,
{
prefix_suffix[0] = prefixlen;
prefix_suffix[1] = suffixlen;
- XLogRegisterBufData(0, &prefix_suffix, sizeof(uint16) * 2);
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW, &prefix_suffix,
+ sizeof(uint16) * 2);
}
else if (prefixlen > 0)
{
- XLogRegisterBufData(0, &prefixlen, sizeof(uint16));
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW, &prefixlen,
+ sizeof(uint16));
}
else
{
- XLogRegisterBufData(0, &suffixlen, sizeof(uint16));
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW, &suffixlen,
+ sizeof(uint16));
}
}
@@ -9009,10 +9016,10 @@ log_heap_update(Relation reln, Buffer oldbuf,
*
* The 'data' doesn't include the common prefix or suffix.
*/
- XLogRegisterBufData(0, &xlhdr, SizeOfHeapHeader);
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW, &xlhdr, SizeOfHeapHeader);
if (prefixlen == 0)
{
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW,
(char *) newtup->t_data + SizeofHeapTupleHeader,
newtup->t_len - SizeofHeapTupleHeader - suffixlen);
}
@@ -9025,13 +9032,13 @@ log_heap_update(Relation reln, Buffer oldbuf,
/* bitmap [+ padding] [+ oid] */
if (newtup->t_data->t_hoff - SizeofHeapTupleHeader > 0)
{
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW,
(char *) newtup->t_data + SizeofHeapTupleHeader,
newtup->t_data->t_hoff - SizeofHeapTupleHeader);
}
/* data after common prefix */
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_NEW,
(char *) newtup->t_data + newtup->t_data->t_hoff + prefixlen,
newtup->t_len - newtup->t_data->t_hoff - prefixlen - suffixlen);
}
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index eb4bd3d6ae3..6cc3bc991c3 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -350,7 +350,8 @@ heap_xlog_delete(XLogReaderState *record)
RelFileLocator target_locator;
ItemPointerData target_tid;
- XLogRecGetBlockTag(record, 0, &target_locator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_DELETE_BLKREF_HEAP, &target_locator, NULL,
+ &blkno);
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
@@ -369,7 +370,8 @@ heap_xlog_delete(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_DELETE_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = BufferGetPage(buffer);
@@ -434,7 +436,8 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerData target_tid;
XLogRedoAction action;
- XLogRecGetBlockTag(record, 0, &target_locator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_INSERT_BLKREF_HEAP, &target_locator, NULL,
+ &blkno);
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
@@ -462,13 +465,14 @@ heap_xlog_insert(XLogReaderState *record)
*/
if (XLogRecGetInfo(record) & XLOG_HEAP_INIT_PAGE)
{
- buffer = XLogInitBufferForRedo(record, 0);
+ buffer = XLogInitBufferForRedo(record, HEAP_INSERT_BLKREF_HEAP);
page = BufferGetPage(buffer);
PageInit(page, BufferGetPageSize(buffer), 0);
action = BLK_NEEDS_REDO;
}
else
- action = XLogReadBufferForRedo(record, 0, &buffer);
+ action = XLogReadBufferForRedo(record, HEAP_INSERT_BLKREF_HEAP,
+ &buffer);
if (action == BLK_NEEDS_REDO)
{
Size datalen;
@@ -479,7 +483,7 @@ heap_xlog_insert(XLogReaderState *record)
if (PageGetMaxOffsetNumber(page) + 1 < xlrec->offnum)
elog(PANIC, "invalid max offset number");
- data = XLogRecGetBlockData(record, 0, &datalen);
+ data = XLogRecGetBlockData(record, HEAP_INSERT_BLKREF_HEAP, &datalen);
newlen = datalen - SizeOfHeapHeader;
Assert(datalen > SizeOfHeapHeader && newlen <= MaxHeapTupleSize);
@@ -559,7 +563,8 @@ heap_xlog_multi_insert(XLogReaderState *record)
*/
xlrec = (xl_heap_multi_insert *) XLogRecGetData(record);
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_MULTI_INSERT_BLKREF_HEAP, &rlocator, NULL,
+ &blkno);
/* check that the mutually exclusive flags are not both set */
Assert(!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
@@ -582,13 +587,14 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (isinit)
{
- buffer = XLogInitBufferForRedo(record, 0);
+ buffer = XLogInitBufferForRedo(record, HEAP_MULTI_INSERT_BLKREF_HEAP);
page = BufferGetPage(buffer);
PageInit(page, BufferGetPageSize(buffer), 0);
action = BLK_NEEDS_REDO;
}
else
- action = XLogReadBufferForRedo(record, 0, &buffer);
+ action = XLogReadBufferForRedo(record, HEAP_MULTI_INSERT_BLKREF_HEAP,
+ &buffer);
if (action == BLK_NEEDS_REDO)
{
char *tupdata;
@@ -596,7 +602,8 @@ heap_xlog_multi_insert(XLogReaderState *record)
Size len;
/* Tuples are stored as block data */
- tupdata = XLogRecGetBlockData(record, 0, &len);
+ tupdata = XLogRecGetBlockData(record, HEAP_MULTI_INSERT_BLKREF_HEAP,
+ &len);
endptr = tupdata + len;
page = (Page) BufferGetPage(buffer);
@@ -713,8 +720,10 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
oldtup.t_data = NULL;
oldtup.t_len = 0;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &newblk);
- if (XLogRecGetBlockTagExtended(record, 1, NULL, NULL, &oldblk, NULL))
+ XLogRecGetBlockTag(record, HEAP_UPDATE_BLKREF_NEW, &rlocator, NULL,
+ &newblk);
+ if (XLogRecGetBlockTagExtended(record, HEAP_UPDATE_BLKREF_OLD, NULL, NULL,
+ &oldblk, NULL))
{
/* HOT updates are never done across pages */
Assert(!hot_update);
@@ -750,7 +759,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
*/
/* Deal with old tuple version */
- oldaction = XLogReadBufferForRedo(record, (oldblk == newblk) ? 0 : 1,
+ oldaction = XLogReadBufferForRedo(record, (oldblk == newblk) ?
+ HEAP_UPDATE_BLKREF_NEW : HEAP_UPDATE_BLKREF_OLD,
&obuffer);
if (oldaction == BLK_NEEDS_REDO)
{
@@ -800,13 +810,14 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
}
else if (XLogRecGetInfo(record) & XLOG_HEAP_INIT_PAGE)
{
- nbuffer = XLogInitBufferForRedo(record, 0);
+ nbuffer = XLogInitBufferForRedo(record, HEAP_UPDATE_BLKREF_NEW);
page = (Page) BufferGetPage(nbuffer);
PageInit(page, BufferGetPageSize(nbuffer), 0);
newaction = BLK_NEEDS_REDO;
}
else
- newaction = XLogReadBufferForRedo(record, 0, &nbuffer);
+ newaction = XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_NEW,
+ &nbuffer);
/*
* The visibility map may need to be fixed even if the heap page is
@@ -831,7 +842,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Size datalen;
Size tuplen;
- recdata = XLogRecGetBlockData(record, 0, &datalen);
+ recdata = XLogRecGetBlockData(record, HEAP_UPDATE_BLKREF_NEW,
+ &datalen);
recdata_end = recdata + datalen;
page = BufferGetPage(nbuffer);
@@ -1015,7 +1027,8 @@ heap_xlog_lock(XLogReaderState *record)
BlockNumber block;
Relation reln;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
+ XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
+ &block);
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
@@ -1025,7 +1038,8 @@ heap_xlog_lock(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = (Page) BufferGetPage(buffer);
@@ -1091,7 +1105,8 @@ heap_xlog_lock_updated(XLogReaderState *record)
BlockNumber block;
Relation reln;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
+ XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
+ &block);
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
@@ -1101,7 +1116,8 @@ heap_xlog_lock_updated(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = BufferGetPage(buffer);
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 277df6b3cf0..55e3c7b0015 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -110,6 +110,9 @@
(XLH_DELETE_CONTAINS_OLD_TUPLE | XLH_DELETE_CONTAINS_OLD_KEY)
/* This is what we need to know about delete */
+#define HEAP_DELETE_BLKREF_HEAP 0
+#define HEAP_DELETE_BLKREF_VM 1
+
typedef struct xl_heap_delete
{
TransactionId xmax; /* xmax of the deleted tuple */
@@ -157,12 +160,15 @@ typedef struct xl_heap_header
#define SizeOfHeapHeader (offsetof(xl_heap_header, t_hoff) + sizeof(uint8))
/* This is what we need to know about insert */
+#define HEAP_INSERT_BLKREF_HEAP 0
+#define HEAP_INSERT_BLKREF_VM 1
+
typedef struct xl_heap_insert
{
OffsetNumber offnum; /* inserted tuple's offset */
uint8 flags;
- /* xl_heap_header & TUPLE DATA in backup block 0 */
+ /* xl_heap_header & TUPLE DATA in HEAP_INSERT_BLKREF_HEAP */
} xl_heap_insert;
#define SizeOfHeapInsert (offsetof(xl_heap_insert, flags) + sizeof(uint8))
@@ -173,10 +179,14 @@ typedef struct xl_heap_insert
* The main data of the record consists of this xl_heap_multi_insert header.
* 'offsets' array is omitted if the whole page is reinitialized
* (XLOG_HEAP_INIT_PAGE).
- *
- * In block 0's data portion, there is an xl_multi_insert_tuple struct,
- * followed by the tuple data for each tuple. There is padding to align
- * each xl_multi_insert_tuple struct.
+ */
+#define HEAP_MULTI_INSERT_BLKREF_HEAP 0
+#define HEAP_MULTI_INSERT_BLKREF_VM 1
+
+/*
+ * In HEAP_MULTI_INSERT_BLKREF_HEAP's data portion, there is an
+ * xl_multi_insert_tuple struct, followed by the tuple data for each tuple.
+ * There is padding to align each xl_multi_insert_tuple struct.
*/
typedef struct xl_heap_multi_insert
{
@@ -201,7 +211,7 @@ typedef struct xl_multi_insert_tuple
/*
* This is what we need to know about update|hot_update
*
- * Backup blk 0: new page
+ * HEAP_UPDATE_BLKREF_NEW: new page
*
* If XLH_UPDATE_PREFIX_FROM_OLD or XLH_UPDATE_SUFFIX_FROM_OLD flags are set,
* the prefix and/or suffix come first, as one or two uint16s.
@@ -213,8 +223,13 @@ typedef struct xl_multi_insert_tuple
* If XLH_UPDATE_CONTAINS_NEW_TUPLE flag is given, the tuple data is
* included even if a full-page image was taken.
*
- * Backup blk 1: old page, if different. (no data, just a reference to the blk)
+ * HEAP_UPDATE_BLKREF_OLD: old page, if different. (no data, just a reference
+ * to the block)
*/
+
+#define HEAP_UPDATE_BLKREF_NEW 0
+#define HEAP_UPDATE_BLKREF_OLD 1
+
typedef struct xl_heap_update
{
TransactionId old_xmax; /* xmax of the old tuple */
@@ -393,6 +408,9 @@ typedef struct xlhp_prune_items
#define XLH_LOCK_ALL_FROZEN_CLEARED 0x01
/* This is what we need to know about lock */
+#define HEAP_LOCK_BLKREF_HEAP 0
+#define HEAP_LOCK_BLKREF_VM 1
+
typedef struct xl_heap_lock
{
TransactionId xmax; /* might be a MultiXactId */
--
2.43.0
v3-0002-Fix-WAL-logging-of-VM-clears.patch (text/x-patch, 40.8K) download
From 9358530be17b2e6d84bb1956dc09afadd817e5df Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 12:53:49 -0400
Subject: [PATCH v3 2/4] Fix WAL logging of VM clears
Previously, we failed to register the visibility map buffer when
emitting WAL after clearing the VM bits for heap pages (recovery code
read the VM page normally). This meant that we couldn't emit FPIs of VM
pages when clearing VM bits. While this would not result in a checksum
error because the visibility map is read with ZERO_ON_ERROR, the WAL
summarizer only includes pages which were registered, so a restore from
an incremental backup would not include the modified VM pages -- leading
to data corruption.
Fix this by registering the VM buffer in the WAL record when clearing VM
bits. This also means the VM buffer must be locked throughout the
duration of the critical section where we make changes to the VM and
heap page and emit WAL.
Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/flat/CAAKRu_bn%2Be7F4yPFBgFbnP%2BsyJRKyNK092bjD2LKvZW7O4Svag>
Backpatch-through: 17
---
contrib/pg_surgery/heap_surgery.c | 39 ++-
src/backend/access/heap/heapam.c | 358 ++++++++++++++++++++----
src/backend/access/heap/heapam_xlog.c | 219 ++++++++++-----
src/backend/access/heap/visibilitymap.c | 23 +-
src/bin/pg_walsummary/t/002_blocks.pl | 7 +-
src/include/access/heapam_xlog.h | 17 +-
src/include/access/visibilitymap.h | 2 +
7 files changed, 520 insertions(+), 145 deletions(-)
diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c
index 602aca66c60..2f89fc1159f 100644
--- a/contrib/pg_surgery/heap_surgery.c
+++ b/contrib/pg_surgery/heap_surgery.c
@@ -17,6 +17,7 @@
#include "access/visibilitymap.h"
#include "access/xloginsert.h"
#include "catalog/pg_am_d.h"
+#include "catalog/pg_control.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
#include "utils/acl.h"
@@ -146,6 +147,7 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
{
Buffer buf;
Buffer vmbuf = InvalidBuffer;
+ bool unlock_vmbuf = false;
Page page;
BlockNumber blkno;
OffsetNumber curoff;
@@ -233,11 +235,15 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
}
/*
- * Before entering the critical section, pin the visibility map page
- * if it appears to be necessary.
+ * Before entering the critical section, pin and lock the visibility
+ * map page if it appears to be necessary.
*/
if (heap_force_opt == HEAP_FORCE_KILL && PageIsAllVisible(page))
+ {
visibilitymap_pin(rel, blkno, &vmbuf);
+ LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuf = true;
+ }
/* No ereport(ERROR) from here until all the changes are logged. */
START_CRIT_SECTION();
@@ -266,10 +272,11 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
*/
if (PageIsAllVisible(page))
{
+ if (visibilitymap_clear_locked(rel, blkno, vmbuf,
+ VISIBILITYMAP_VALID_BITS))
+ did_modify_vm = true;
+
PageClearAllVisible(page);
- visibilitymap_clear(rel, blkno, vmbuf,
- VISIBILITYMAP_VALID_BITS);
- did_modify_vm = true;
}
}
else
@@ -320,18 +327,28 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
/* XLOG stuff */
if (RelationNeedsWAL(rel))
- log_newpage_buffer(buf, true);
+ {
+ XLogRecPtr recptr;
+
+ XLogBeginInsert();
+ XLogRegisterBuffer(0, buf, REGBUF_STANDARD | REGBUF_FORCE_IMAGE);
+ if (did_modify_vm)
+ XLogRegisterBuffer(1, vmbuf, REGBUF_FORCE_IMAGE);
+ recptr = XLogInsert(RM_XLOG_ID, XLOG_FPI);
+ if (did_modify_vm)
+ PageSetLSN(BufferGetPage(vmbuf), recptr);
+ PageSetLSN(BufferGetPage(buf), recptr);
+ }
}
- /* WAL log the VM page if it was modified. */
- if (did_modify_vm && RelationNeedsWAL(rel))
- log_newpage_buffer(vmbuf, false);
-
END_CRIT_SECTION();
UnlockReleaseBuffer(buf);
- if (vmbuf != InvalidBuffer)
+ if (unlock_vmbuf)
+ LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
+
+ if (BufferIsValid(vmbuf))
ReleaseBuffer(vmbuf);
/* Update the current_start_ptr before moving to the next page. */
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 3c3b65b3cbf..7dfa3221652 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -58,7 +58,8 @@
static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup,
TransactionId xid, CommandId cid, int options);
static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup,
+ Buffer vmbuffer_old, Buffer newbuf,
+ Buffer vmbuffer_new, HeapTuple oldtup,
HeapTuple newtup, HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared);
#ifdef USE_ASSERT_CHECKING
@@ -2083,8 +2084,10 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
bool all_visible_cleared = false;
+ bool vmbuffer_modified = false;
/* Cheap, simplistic check that the tuple matches the rel's rowtype. */
Assert(HeapTupleHeaderGetNatts(tup->t_data) <=
@@ -2108,6 +2111,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
InvalidBuffer, options, bistate,
&vmbuffer, NULL,
0);
+ page = BufferGetPage(buffer);
/*
* We're about to do the actual insert -- but check for conflict first, to
@@ -2126,19 +2130,28 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
*/
CheckForSerializableConflictIn(relation, NULL, InvalidBlockNumber);
+ /* Lock the vmbuffer before the critical section */
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ all_visible_cleared = true;
+ }
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ if (all_visible_cleared)
{
- all_visible_cleared = true;
- PageClearAllVisible(BufferGetPage(buffer));
- visibilitymap_clear(relation,
- ItemPointerGetBlockNumber(&(heaptup->t_self)),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ /* It's possible the VM bits were already clear */
+ if (visibilitymap_clear_locked(relation,
+ ItemPointerGetBlockNumber(&(heaptup->t_self)),
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_modified = true;
+
+ PageClearAllVisible(page);
}
/*
@@ -2160,7 +2173,6 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xl_heap_insert xlrec;
xl_heap_header xlhdr;
XLogRecPtr recptr;
- Page page = BufferGetPage(buffer);
uint8 info = XLOG_HEAP_INSERT;
int bufflags = 0;
@@ -2230,15 +2242,28 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (vmbuffer_modified)
+ XLogRegisterBuffer(HEAP_INSERT_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, info);
PageSetLSN(page, recptr);
+
+ if (vmbuffer_modified)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
+
+ /*
+ * We locked vmbuffer if all_visible_cleared was true regardless of
+ * whether or not we ended up modifying it.
+ */
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ if (BufferIsValid(vmbuffer))
ReleaseBuffer(vmbuffer);
/*
@@ -2421,6 +2446,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
Buffer buffer;
bool all_visible_cleared = false;
bool all_frozen_set = false;
+ bool vmbuffer_modified = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2462,6 +2488,17 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
all_frozen_set = true;
+ /*
+ * If clearing all-visible, take the VM buffer lock before entering
+ * the critical section where that action will be WAL-logged. Setting
+ * the VM all-frozen is done and WAL-logged separately.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ all_visible_cleared = true;
+ }
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2502,13 +2539,14 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If we're only adding already frozen rows to a previously empty
* page, mark it as all-visible.
*/
- if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
+ if (all_visible_cleared)
{
- all_visible_cleared = true;
+ if (visibilitymap_clear_locked(relation,
+ BufferGetBlockNumber(buffer),
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_modified = true;
+
PageClearAllVisible(page);
- visibilitymap_clear(relation,
- BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
}
else if (all_frozen_set)
PageSetAllVisible(page);
@@ -2623,6 +2661,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogRegisterData(xlrec, tupledata - scratch.data);
XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_HEAP, buffer,
REGBUF_STANDARD | bufflags);
+ if (vmbuffer_modified)
+ XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_VM, vmbuffer, 0);
XLogRegisterBufData(HEAP_MULTI_INSERT_BLKREF_HEAP, tupledata,
totaldatalen);
@@ -2633,10 +2673,19 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
recptr = XLogInsert(RM_HEAP2_ID, info);
PageSetLSN(page, recptr);
+ if (vmbuffer_modified)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /*
+ * We locked vmbuffer if all_visible_cleared was true regardless of
+ * whether or not we ended up modifying it.
+ */
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
/*
* If we've frozen everything on the page, update the visibilitymap.
* We're already holding pin on the vmbuffer.
@@ -2786,6 +2835,7 @@ heap_delete(Relation relation, ItemPointer tid,
BlockNumber block;
Buffer buffer;
Buffer vmbuffer = InvalidBuffer;
+ bool vmbuffer_modified = false;
TransactionId new_xmax;
uint16 new_infomask,
new_infomask2;
@@ -3040,6 +3090,13 @@ l1:
xid, LockTupleExclusive, true,
&new_xmax, &new_infomask, &new_infomask2);
+ /* Lock the VM before entering the critical section */
+ if (PageIsAllVisible(page))
+ {
+ all_visible_cleared = true;
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ }
+
START_CRIT_SECTION();
/*
@@ -3051,12 +3108,14 @@ l1:
*/
PageSetPrunable(page, xid);
- if (PageIsAllVisible(page))
+ if (all_visible_cleared)
{
- all_visible_cleared = true;
+ /* It's possible the VM bits were already clear */
+ if (visibilitymap_clear_locked(relation, BufferGetBlockNumber(buffer),
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_modified = true;
+
PageClearAllVisible(page);
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
}
/* store transaction information of xact deleting the tuple */
@@ -3137,13 +3196,27 @@ l1:
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (vmbuffer_modified)
+ XLogRegisterBuffer(HEAP_DELETE_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE);
PageSetLSN(page, recptr);
+
+ if (vmbuffer_modified)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /*
+ * Release VM lock first, since it covers many heap blocks. We locked
+ * vmbuffer if all_visible_cleared was true regardless of whether or not
+ * we ended up modifying it.
+ */
+ if (all_visible_cleared)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
if (vmbuffer != InvalidBuffer)
@@ -3261,13 +3334,16 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
HeapTuple heaptup;
HeapTuple old_key_tuple = NULL;
bool old_key_copied = false;
- Page page;
+ Page page,
+ newpage;
BlockNumber block;
MultiXactStatus mxact_status;
Buffer buffer,
newbuf,
vmbuffer = InvalidBuffer,
vmbuffer_new = InvalidBuffer;
+ bool unlock_vmbuffer = false;
+ bool unlock_vmbuffer_new = false;
bool need_toast;
Size newtupsize,
pagefree;
@@ -3278,6 +3354,8 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
bool key_intact;
bool all_visible_cleared = false;
bool all_visible_cleared_new = false;
+ bool vmbuffer_modified = false;
+ bool vmbuffer_new_modified = false;
bool checked_lockers;
bool locker_remains;
bool id_has_external = false;
@@ -3852,6 +3930,12 @@ l2:
Assert(HEAP_XMAX_IS_LOCKED_ONLY(infomask_lock_old_tuple));
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuffer = true;
+ }
+
START_CRIT_SECTION();
/* Clear obsolete visibility flags ... */
@@ -3874,10 +3958,12 @@ l2:
* overhead would be unchanged, that doesn't seem necessarily
* worthwhile.
*/
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ if (visibilitymap_clear_locked(relation, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ }
MarkBufferDirty(buffer);
@@ -3896,12 +3982,24 @@ l2:
xlrec.flags =
cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (unlock_vmbuffer)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ unlock_vmbuffer = false;
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
/*
@@ -3987,6 +4085,8 @@ l2:
heaptup = newtup;
}
+ newpage = BufferGetPage(newbuf);
+
/*
* We're about to do the actual update -- check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -4050,6 +4150,69 @@ l2:
id_has_external,
&old_key_copied);
+ all_visible_cleared = PageIsAllVisible(page);
+ all_visible_cleared_new = newbuf != buffer && PageIsAllVisible(newpage);
+
+ /*
+ * Clear PD_ALL_VISIBLE flags and reset visibility map bits for any heap
+ * pages that were all-visible. If there are two heap pages, we may need
+ * to clear VM bits for both.
+ */
+ if (all_visible_cleared && all_visible_cleared_new &&
+ vmbuffer_new == vmbuffer)
+ {
+ /*
+ * This is the more complicated case: both the new and old heap pages
+ * are all-visible and both their VM bits are on the same page of the
+ * VM, so we register a single VM buffer as HEAP_UPDATE_BLKREF_VM_NEW
+ * in the WAL record. We must be careful to only lock and register one
+ * buffer, even though we modify it twice -- once for each heap
+ * block's VM bits.
+ */
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuffer_new = true;
+
+ /* We will not lock or attempt to modify old VM buffer */
+ }
+ else
+ {
+ /*
+ * In all the remaining cases, we will clear at most one heap block's
+ * VM bits per VM page.
+ */
+ Buffer vmbuffers[2] = {
+ all_visible_cleared ? vmbuffer : InvalidBuffer,
+ all_visible_cleared_new ? vmbuffer_new : InvalidBuffer
+ };
+
+ /*
+ * When both pages need different VM pages cleared, acquire the VM
+ * buffer locks in VM block order to avoid deadlocks between backends
+ * updating tuples in opposite directions across VM pages.
+ */
+ if (all_visible_cleared && all_visible_cleared_new &&
+ BufferGetBlockNumber(vmbuffers[0]) > BufferGetBlockNumber(vmbuffers[1]))
+ {
+ Buffer swap = vmbuffers[0];
+
+ vmbuffers[0] = vmbuffers[1];
+ vmbuffers[1] = swap;
+ }
+
+ Assert((!BufferIsValid(vmbuffers[0]) && !BufferIsValid(vmbuffers[1])) ||
+ vmbuffers[0] != vmbuffers[1]);
+
+ if (BufferIsValid(vmbuffers[0]))
+ LockBuffer(vmbuffers[0], BUFFER_LOCK_EXCLUSIVE);
+ if (BufferIsValid(vmbuffers[1]))
+ LockBuffer(vmbuffers[1], BUFFER_LOCK_EXCLUSIVE);
+
+ if (all_visible_cleared)
+ unlock_vmbuffer = true;
+ if (all_visible_cleared_new)
+ unlock_vmbuffer_new = true;
+ }
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -4086,6 +4249,34 @@ l2:
RelationPutHeapTuple(relation, newbuf, heaptup, false); /* insert new tuple */
+ if (all_visible_cleared)
+ {
+ if (visibilitymap_clear_locked(relation, block,
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ {
+ /*
+ * If both heap block's VM bits are on the same VM buffer,
+ * vmbuffer_new and vmbuffer will be equal. In that case only
+ * vmbuffer_new is registered in the WAL record (see
+ * log_heap_update), so attribute any change to the old heap
+ * block's VM bits to vmbuffer_new as well.
+ */
+ if (vmbuffer == vmbuffer_new)
+ vmbuffer_new_modified = true;
+ else
+ vmbuffer_modified = true;
+ }
+
+ PageClearAllVisible(page);
+ }
+ if (all_visible_cleared_new)
+ {
+ if (visibilitymap_clear_locked(relation, BufferGetBlockNumber(newbuf),
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_new_modified = true;
+
+ PageClearAllVisible(newpage);
+ }
/* Clear obsolete visibility flags, possibly set by ourselves above... */
oldtup.t_data->t_infomask &= ~(HEAP_XMAX_BITS | HEAP_MOVED);
@@ -4100,21 +4291,6 @@ l2:
/* record address of new tuple in t_ctid of old one */
oldtup.t_data->t_ctid = heaptup->t_self;
- /* clear PD_ALL_VISIBLE flags, reset all visibilitymap bits */
- if (PageIsAllVisible(BufferGetPage(buffer)))
- {
- all_visible_cleared = true;
- PageClearAllVisible(BufferGetPage(buffer));
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
- }
- if (newbuf != buffer && PageIsAllVisible(BufferGetPage(newbuf)))
- {
- all_visible_cleared_new = true;
- PageClearAllVisible(BufferGetPage(newbuf));
- visibilitymap_clear(relation, BufferGetBlockNumber(newbuf),
- vmbuffer_new, VISIBILITYMAP_VALID_BITS);
- }
if (newbuf != buffer)
MarkBufferDirty(newbuf);
@@ -4136,19 +4312,30 @@ l2:
}
recptr = log_heap_update(relation, buffer,
- newbuf, &oldtup, heaptup,
+ vmbuffer_modified ? vmbuffer : InvalidBuffer,
+ newbuf,
+ vmbuffer_new_modified ? vmbuffer_new : InvalidBuffer,
+ &oldtup, heaptup,
old_key_tuple,
all_visible_cleared,
all_visible_cleared_new);
if (newbuf != buffer)
- {
- PageSetLSN(BufferGetPage(newbuf), recptr);
- }
+ PageSetLSN(newpage, recptr);
PageSetLSN(BufferGetPage(buffer), recptr);
+
+ if (vmbuffer_modified)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
+ if (vmbuffer_new_modified)
+ PageSetLSN(BufferGetPage(vmbuffer_new), recptr);
}
END_CRIT_SECTION();
+ if (unlock_vmbuffer)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ if (unlock_vmbuffer_new)
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_UNLOCK);
+
if (newbuf != buffer)
LockBuffer(newbuf, BUFFER_LOCK_UNLOCK);
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
@@ -4586,6 +4773,7 @@ heap_lock_tuple(Relation relation, HeapTuple tuple,
ItemId lp;
Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool unlock_vmbuffer = false;
BlockNumber block;
TransactionId xid,
xmax;
@@ -5166,6 +5354,13 @@ failed:
GetCurrentTransactionId(), mode, false,
&xid, &new_infomask, &new_infomask2);
+ /* Lock VM buffer before entering critical section */
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuffer = true;
+ }
+
START_CRIT_SECTION();
/*
@@ -5197,11 +5392,12 @@ failed:
tuple->t_data->t_ctid = *tid;
/* Clear only the all-frozen bit on visibility map if needed */
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
-
+ if (PageIsAllVisible(page))
+ {
+ if (visibilitymap_clear_locked(relation, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ }
MarkBufferDirty(*buffer);
@@ -5232,15 +5428,25 @@ failed:
xlrec.flags = cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
/* we don't decode row locks atm, so no need to log the origin */
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (unlock_vmbuffer)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
result = TM_Ok;
out_locked:
@@ -5707,6 +5913,7 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
ItemPointerData tupid;
HeapTupleData mytup;
Buffer buf;
+ Page page;
uint16 new_infomask,
new_infomask2,
old_infomask,
@@ -5716,6 +5923,7 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
bool cleared_all_frozen = false;
bool pinned_desired_page;
Buffer vmbuffer = InvalidBuffer;
+ bool unlock_vmbuffer = false;
BlockNumber block;
ItemPointerCopy(tid, &tupid);
@@ -5724,6 +5932,7 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
{
new_infomask = 0;
new_xmax = InvalidTransactionId;
+ cleared_all_frozen = false;
block = ItemPointerGetBlockNumber(&tupid);
ItemPointerCopy(&tupid, &(mytup.t_self));
@@ -5743,13 +5952,15 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
l4:
CHECK_FOR_INTERRUPTS();
+ page = BufferGetPage(buf);
+
/*
* Before locking the buffer, pin the visibility map page if it
* appears to be necessary. Since we haven't got the lock yet,
* someone else might be in the middle of changing this, so we'll need
* to recheck after we have the lock.
*/
- if (PageIsAllVisible(BufferGetPage(buf)))
+ if (PageIsAllVisible(page))
{
visibilitymap_pin(rel, block, &vmbuffer);
pinned_desired_page = true;
@@ -5770,7 +5981,7 @@ l4:
* this page. If this page isn't all-visible, we won't use the vm
* page, but we hold onto such a pin till the end of the function.
*/
- if (!pinned_desired_page && PageIsAllVisible(BufferGetPage(buf)))
+ if (!pinned_desired_page && PageIsAllVisible(page))
{
LockBuffer(buf, BUFFER_LOCK_UNLOCK);
visibilitymap_pin(rel, block, &vmbuffer);
@@ -5951,10 +6162,11 @@ l4:
xid, mode, false,
&new_xmax, &new_infomask, &new_infomask2);
- if (PageIsAllVisible(BufferGetPage(buf)) &&
- visibilitymap_clear(rel, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuffer = true;
+ }
START_CRIT_SECTION();
@@ -5967,12 +6179,18 @@ l4:
MarkBufferDirty(buf);
+ if (PageIsAllVisible(page))
+ {
+ if (visibilitymap_clear_locked(rel, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ }
+
/* XLOG stuff */
if (RelationNeedsWAL(rel))
{
xl_heap_lock_updated xlrec;
XLogRecPtr recptr;
- Page page = BufferGetPage(buf);
XLogBeginInsert();
XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buf, REGBUF_STANDARD);
@@ -5985,13 +6203,26 @@ l4:
XLogRegisterData(&xlrec, SizeOfHeapLockUpdated);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_LOCK_UPDATED);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (unlock_vmbuffer)
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ unlock_vmbuffer = false;
+ }
+
next:
/* if we find the end of update chain, we're done. */
if (mytup.t_data->t_infomask & HEAP_XMAX_INVALID ||
@@ -8848,8 +9079,9 @@ log_heap_visible(Relation rel, Buffer heap_buffer, Buffer vm_buffer,
* have modified the buffer(s) and marked them dirty.
*/
static XLogRecPtr
-log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup, HeapTuple newtup,
+log_heap_update(Relation reln, Buffer oldbuf, Buffer vmbuffer_old,
+ Buffer newbuf, Buffer vmbuffer_new,
+ HeapTuple oldtup, HeapTuple newtup,
HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared)
{
@@ -9058,6 +9290,20 @@ log_heap_update(Relation reln, Buffer oldbuf,
old_key_tuple->t_len - SizeofHeapTupleHeader);
}
+ /*
+ * Register VM buffers. If the old and new heap pages' VM bits are on the
+ * same VM page, the caller passes only vmbuffer_new (mirroring the heap
+ * page convention where block 0 = new is always registered).
+ */
+ Assert((BufferIsInvalid(vmbuffer_old) && BufferIsInvalid(vmbuffer_new)) ||
+ (vmbuffer_old != vmbuffer_new));
+
+ if (BufferIsValid(vmbuffer_new))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_NEW, vmbuffer_new, 0);
+
+ if (BufferIsValid(vmbuffer_old))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_OLD, vmbuffer_old, 0);
+
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index 6cc3bc991c3..1b546268a20 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -22,6 +22,51 @@
#include "storage/freespace.h"
#include "storage/standby.h"
+/*
+ * Helper to clear visibility map bits during heap redo.
+ *
+ * This handles records that modify one heap block and its corresponding VM
+ * block. More complex cases, such as update records that can touch multiple
+ * heap or VM blocks, must handle VM replay directly. 'flags' specifies which
+ * visibility map bits to clear.
+ */
+static void
+heap_xlog_vm_clear(XLogReaderState *record,
+ XLogRecPtr lsn, RelFileLocator target_locator,
+ BlockNumber heap_blkno,
+ int vm_block_id, uint8 flags)
+{
+ Relation reln = CreateFakeRelcacheEntry(target_locator);
+ Buffer vmbuffer = InvalidBuffer;
+
+ /*
+ * If this record includes the VM block, use it for redo. Older releases
+ * did not register the VM buffer when clearing the VM, so keep the
+ * fallback path to support replay of WAL generated before that fix.
+ */
+ if (XLogRecHasBlockRef(record, vm_block_id))
+ {
+ if (XLogReadBufferForRedo(record, vm_block_id,
+ &vmbuffer) == BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln,
+ heap_blkno, vmbuffer,
+ flags);
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else
+ {
+ visibilitymap_pin(reln, heap_blkno, &vmbuffer);
+ visibilitymap_clear(reln, heap_blkno, vmbuffer, flags);
+ ReleaseBuffer(vmbuffer);
+ }
+
+ FreeFakeRelcacheEntry(reln);
+}
+
/*
* Replay XLOG_HEAP2_PRUNE_* records.
@@ -360,15 +405,9 @@ heap_xlog_delete(XLogReaderState *record)
* already up-to-date.
*/
if (xlrec->flags & XLH_DELETE_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(target_locator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
+ heap_xlog_vm_clear(record, lsn, target_locator,
+ blkno, HEAP_DELETE_BLKREF_VM,
+ VISIBILITYMAP_VALID_BITS);
if (XLogReadBufferForRedo(record, HEAP_DELETE_BLKREF_HEAP,
&buffer) == BLK_NEEDS_REDO)
@@ -449,15 +488,9 @@ heap_xlog_insert(XLogReaderState *record)
* already up-to-date.
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(target_locator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
+ heap_xlog_vm_clear(record, lsn, target_locator,
+ blkno, HEAP_INSERT_BLKREF_VM,
+ VISIBILITYMAP_VALID_BITS);
/*
* If we inserted the first and only tuple on the page, re-initialize the
@@ -571,19 +604,19 @@ heap_xlog_multi_insert(XLogReaderState *record)
(xlrec->flags & XLH_INSERT_ALL_FROZEN_SET)));
/*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
+ * If required, clear the VM first, to prevent all-visible temporarily
+ * being set for a heap page that's not all visible anymore.
+ *
+ * If it were possible for XLH_INSERT_ALL_VISIBLE_CLEARED and
+ * XLH_INSERT_ALL_FROZEN_SET to be present in the same record, doing the
+ * XLogReadBufferForRedo() before the PageSetAllVisible() below would be a
+ * problem, as it'd violate the rule that a heap page must never be set
+ * all-visible in the VM while its PD_ALL_VISIBLE is clear.
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
+ heap_xlog_vm_clear(record, lsn, rlocator,
+ blkno, HEAP_MULTI_INSERT_BLKREF_VM,
+ VISIBILITYMAP_VALID_BITS);
if (isinit)
{
@@ -698,6 +731,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Buffer obuffer,
nbuffer;
Page page;
+ bool new_cleared,
+ old_cleared;
OffsetNumber offnum;
ItemId lp = NULL;
HeapTupleData oldtup;
@@ -737,14 +772,85 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
*/
- if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED)
+ new_cleared = (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0;
+ old_cleared = (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) != 0;
+ if (new_cleared || old_cleared)
{
Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
+ bool has_vm_old = XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_OLD);
+ bool has_vm_new = XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_NEW);
+
+ if (has_vm_new)
+ {
+ Buffer vmbuffer_new = InvalidBuffer;
+
+ Assert(new_cleared);
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW, &vmbuffer_new) ==
+ BLK_NEEDS_REDO)
+ {
+ /*
+ * If both the old and new heap pages were all-visible and
+ * their VM bits are on the same VM page, that single VM page
+ * is registered as HEAP_UPDATE_BLKREF_VM_NEW. Clear both heap
+ * blocks' VM bits from the single provided VM buffer.
+ *
+ * We must verify that oldblk's VM bits really are on this VM
+ * page, rather than relying on the absence of a separate
+ * VM_OLD block reference: VM_OLD is also omitted when oldblk
+ * is on a different VM page but its bit was already clear.
+ */
+ if (old_cleared && visibilitymap_pin_ok(oldblk, vmbuffer_new))
+ visibilitymap_clear_locked(reln, oldblk, vmbuffer_new,
+ VISIBILITYMAP_VALID_BITS);
+ visibilitymap_clear_locked(reln, newblk, vmbuffer_new,
+ VISIBILITYMAP_VALID_BITS);
+ PageSetLSN(BufferGetPage(vmbuffer_new), lsn);
+ }
+ if (BufferIsValid(vmbuffer_new))
+ UnlockReleaseBuffer(vmbuffer_new);
+ }
+ if (has_vm_old)
+ {
+ Buffer vmbuffer_old = InvalidBuffer;
+
+ Assert(old_cleared);
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_OLD, &vmbuffer_old) ==
+ BLK_NEEDS_REDO)
+ {
+ visibilitymap_clear_locked(reln, oldblk, vmbuffer_old,
+ VISIBILITYMAP_VALID_BITS);
+ PageSetLSN(BufferGetPage(vmbuffer_old), lsn);
+ }
+ if (BufferIsValid(vmbuffer_old))
+ UnlockReleaseBuffer(vmbuffer_old);
+ }
+ if (!has_vm_old && !has_vm_new)
+ {
+ /*
+ * Backwards compatibility path. Previously, the VM buffers were
+ * not registered in the WAL record. We need this path to replay
+ * WAL generated by a not-yet-patched primary during upgrade.
+ */
+ if (old_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ visibilitymap_pin(reln, oldblk, &vmbuffer);
+ visibilitymap_clear(reln, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
+ if (new_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ visibilitymap_pin(reln, newblk, &vmbuffer);
+ visibilitymap_clear(reln, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
+ }
- visibilitymap_pin(reln, oldblk, &vmbuffer);
- visibilitymap_clear(reln, oldblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -793,7 +899,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
/* Mark the page as a candidate for pruning */
PageSetPrunable(page, XLogRecGetXid(record));
- if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED)
+ if (old_cleared)
PageClearAllVisible(page);
PageSetLSN(page, lsn);
@@ -819,21 +925,6 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
newaction = XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_NEW,
&nbuffer);
- /*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
- */
- if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, newblk, &vmbuffer);
- visibilitymap_clear(reln, newblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
-
/* Deal with new tuple */
if (newaction == BLK_NEEDS_REDO)
{
@@ -930,7 +1021,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
if (offnum == InvalidOffsetNumber)
elog(PANIC, "failed to add tuple");
- if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED)
+ if (new_cleared)
PageClearAllVisible(page);
freespace = PageGetHeapFreeSpace(page); /* needed to update FSM below */
@@ -1022,20 +1113,15 @@ heap_xlog_lock(XLogReaderState *record)
*/
if (xlrec->flags & XLH_LOCK_ALL_FROZEN_CLEARED)
{
- RelFileLocator rlocator;
- Buffer vmbuffer = InvalidBuffer;
BlockNumber block;
- Relation reln;
+ RelFileLocator rlocator;
XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
&block);
- reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ heap_xlog_vm_clear(record, lsn, rlocator,
+ block, HEAP_LOCK_BLKREF_VM,
+ VISIBILITYMAP_ALL_FROZEN);
}
if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
@@ -1100,20 +1186,15 @@ heap_xlog_lock_updated(XLogReaderState *record)
*/
if (xlrec->flags & XLH_LOCK_ALL_FROZEN_CLEARED)
{
- RelFileLocator rlocator;
- Buffer vmbuffer = InvalidBuffer;
BlockNumber block;
- Relation reln;
+ RelFileLocator rlocator;
XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
&block);
- reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ heap_xlog_vm_clear(record, lsn, rlocator,
+ block, HEAP_LOCK_BLKREF_VM,
+ VISIBILITYMAP_ALL_FROZEN);
}
if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 1874a3fda37..f02778ba58f 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -135,9 +135,27 @@ static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks);
* You must pass a buffer containing the correct map page to this function.
* Call visibilitymap_pin first to pin the right one. This function doesn't do
* any I/O. Returns true if any bits have been cleared and false otherwise.
+ *
+ * This is retained for backwards compatability, as it is usually necessary to
+ * register the VM buffer in the WAL record and this necessitates holding the
+ * lock for longer than it is held here.
*/
bool
visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
+{
+ bool cleared = false;
+
+ LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+
+ cleared = visibilitymap_clear_locked(rel, heapBlk, vmbuf, flags);
+
+ LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
+
+ return cleared;
+}
+
+bool
+visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
{
BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
@@ -157,7 +175,8 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
elog(ERROR, "wrong buffer passed to visibilitymap_clear");
- LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+ Assert(BufferIsExclusiveLocked(vmbuf));
+
map = PageGetContents(BufferGetPage(vmbuf));
if (map[mapByte] & mask)
@@ -168,8 +187,6 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
cleared = true;
}
- LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
-
return cleared;
}
diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl
index 0f98c7df82e..fcb701e1b98 100644
--- a/src/bin/pg_walsummary/t/002_blocks.pl
+++ b/src/bin/pg_walsummary/t/002_blocks.pl
@@ -93,13 +93,14 @@ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
split(m@/@, $end_lsn);
ok(-f $filename, "WAL summary file exists");
-# Run pg_walsummary on it. We expect exactly two blocks to be modified,
-# block 0 and one other.
+# Run pg_walsummary on it. We expect exactly three blocks to be modified,
+# block 0 (old tuple), another block (new tuple), and the block for the VM.
my ($stdout, $stderr) = run_command([ 'pg_walsummary', '-i', $filename ]);
note($stdout);
@lines = split(/\n/, $stdout);
like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified");
+like($stdout, qr/FORK vm: block 0$/m, "stdout shows VM block 0 modified");
is($stderr, '', 'stderr is empty');
-is(0 + @lines, 2, "UPDATE modified 2 blocks");
+is(0 + @lines, 3, "UPDATE modified 3 blocks");
done_testing();
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 55e3c7b0015..63062424dad 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -225,10 +225,21 @@ typedef struct xl_multi_insert_tuple
*
* HEAP_UPDATE_BLKREF_OLD: old page, if different. (no data, just a reference
* to the block)
+ *
+ * HEAP_UPDATE_BLKREF_VM_NEW: VM page covering the new heap page. Registered
+ * when XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED is set and the new heap page's VM
+ * bit was actually cleared. Also covers the old heap page's VM bits when both
+ * heap pages map to the same VM page.
+ *
+ * HEAP_UPDATE_BLKREF_VM_OLD: VM page covering the old heap page. Only
+ * registered when XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED is set, the old heap
+ * page's VM bits are on a different VM page from the new heap page's, and the
+ * old heap page's VM bit was actually cleared.
*/
-
-#define HEAP_UPDATE_BLKREF_NEW 0
-#define HEAP_UPDATE_BLKREF_OLD 1
+#define HEAP_UPDATE_BLKREF_NEW 0
+#define HEAP_UPDATE_BLKREF_OLD 1
+#define HEAP_UPDATE_BLKREF_VM_NEW 2
+#define HEAP_UPDATE_BLKREF_VM_OLD 3
typedef struct xl_heap_update
{
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index ea889bf9ec7..1bc59c9ac33 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -26,6 +26,8 @@
#define VM_ALL_FROZEN(r, b, v) \
((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0)
+extern bool visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk,
+ Buffer vmbuf, uint8 flags);
extern bool visibilitymap_clear(Relation rel, BlockNumber heapBlk,
Buffer vmbuf, uint8 flags);
extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk,
--
2.43.0
v3-0003-Put-pg_combinebackup-debug-test-output-in-temp-fi.patch (text/x-patch, 4.2K) download
From cb8ffb20e629eb3d5c4234a02863abaefa0f21f7 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 30 Apr 2026 16:06:06 -0400
Subject: [PATCH v3 3/4] Put pg_combinebackup debug test output in temp file
Having --debug as the default option to pg_combinebackup in
init_from_backup() makes the logs extremely verbose. It is used for
debugging failures, so put the output in a temp file and only keep it
around if tests fail.
Author: Melanie Plageman <melanieplageman@gmail.com>
Disussion: https://postgr.es/m/y73coty2smkxgvelh4e32kqrt5pyn2pb7cz7x4mrz6yaxitqd5%40nzhkeazqf4bn
Backpatch-through: 17
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 83 +++++++++++++++++++++++-
1 file changed, 81 insertions(+), 2 deletions(-)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 2e31c23a6ac..3388e45b726 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -125,6 +125,8 @@ our $min_compat = 12;
# list of file reservations made by get_free_port
my @port_reservation_files;
+my @combinebackup_debug_logs;
+my %combinebackup_debug_logs_printed;
# We want to choose a server port above the range that servers typically use
# on Unix systems and below the range those systems typically use for ephemeral
@@ -898,7 +900,9 @@ pathnames that should be used for the target cluster.
To restore from an incremental backup, pass the parameter combine_with_prior
as a reference to an array of prior backup names with which this backup
-is to be combined using pg_combinebackup.
+is to be combined using pg_combinebackup. pg_combinebackup is run with
+--debug, but the output is captured in a temporary file and only copied to the
+test log if pg_combinebackup fails or if a later test fails.
Streaming replication can be enabled on this node by passing the keyword
parameter has_streaming => 1. This is disabled by default.
@@ -964,7 +968,7 @@ sub init_from_backup
}
push @combineargs, @prior_backup_path, $backup_path,
'--output' => $data_path;
- PostgreSQL::Test::Utils::system_or_bail(@combineargs);
+ _run_pg_combinebackup_with_debug_log(@combineargs);
}
elsif (defined $params{tar_program})
{
@@ -1079,6 +1083,81 @@ port = $port
return;
}
+sub _run_pg_combinebackup_with_debug_log
+{
+ my (@cmd) = @_;
+ my $debug_log = PostgreSQL::Test::Utils::tempdir('pg_combinebackup') .
+ '/debug.log';
+
+ push @combinebackup_debug_logs, $debug_log;
+
+ print("# Running: " . join(" ", @cmd) . "\n");
+ # If this pg_combinebackup invocation itself fails, print its captured
+ # output immediately. Successful invocations may still be printed later if
+ # some subsequent TAP assertion in this test script fails.
+ if (!IPC::Run::run(\@cmd, '&>' => $debug_log))
+ {
+ _print_pg_combinebackup_debug_log($debug_log);
+
+ if ($? == -1)
+ {
+ BAIL_OUT(
+ sprintf(
+ "failed to execute command \"%s\": $!", join(" ", @cmd)));
+ }
+ elsif ($? & 127)
+ {
+ BAIL_OUT(
+ sprintf(
+ "command \"%s\" died with signal %d",
+ join(" ", @cmd), $? & 127));
+ }
+ else
+ {
+ BAIL_OUT(
+ sprintf(
+ "command \"%s\" exited with value %d",
+ join(" ", @cmd), $? >> 8));
+ }
+ }
+}
+
+sub _print_pg_combinebackup_debug_log
+{
+ my ($debug_log) = @_;
+
+ return unless -e $debug_log;
+ return if $combinebackup_debug_logs_printed{$debug_log}++;
+
+ diag("pg_combinebackup debug output from $debug_log:\n" .
+ PostgreSQL::Test::Utils::slurp_file($debug_log));
+}
+
+# Keep pg_combinebackup debug logs until test exit. If pg_combinebackup itself
+# fails, its log is printed at the failure site above. If pg_combinebackup
+# succeeds but a later TAP assertion fails, print the retained logs here so the
+# regress output includes the combine step that produced the restored state.
+# Remove the logs after a clean run.
+END
+{
+ # take care not to change the script's exit value
+ my $exit_code = $?;
+
+ if ($exit_code == 0 && PostgreSQL::Test::Utils::all_tests_passing())
+ {
+ unlink @combinebackup_debug_logs;
+ }
+ else
+ {
+ foreach my $debug_log (@combinebackup_debug_logs)
+ {
+ _print_pg_combinebackup_debug_log($debug_log);
+ }
+ }
+
+ $? = $exit_code;
+}
+
=pod
=item $node->rotate_logfile()
--
2.43.0
v3-0004-Test-that-VM-clear-registers-VM-buffers.patch (text/x-patch, 7.9K) download
From dd9734f313a4c1410e07a14f0861f53740cec095 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 23 Apr 2026 11:13:23 -0400
Subject: [PATCH v3 4/4] Test that VM clear registers VM buffers
WAL summarizer only includes registered buffers, so you could end up
with data corruption in incremental backups if VM clears were meant to
be covered by the incremental backup.
Author: Melanie Plageman <melanieplageman@gmail.com>
Discussion: https://postgr.es/m/oqcsevg35xjan2327x5kdfth6q4fgeqboxfo3v3imeyih2uiny%406sez5dzxl6nt
Backpatch-through: 17
---
src/bin/pg_combinebackup/meson.build | 1 +
.../pg_combinebackup/t/012_vm_consistency.pl | 205 ++++++++++++++++++
2 files changed, 206 insertions(+)
create mode 100644 src/bin/pg_combinebackup/t/012_vm_consistency.pl
diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build
index bbc4c5735ba..757b78e2fe8 100644
--- a/src/bin/pg_combinebackup/meson.build
+++ b/src/bin/pg_combinebackup/meson.build
@@ -39,6 +39,7 @@ tests += {
't/009_no_full_file.pl',
't/010_hardlink.pl',
't/011_ib_truncation.pl',
+ 't/012_vm_consistency.pl',
],
}
}
diff --git a/src/bin/pg_combinebackup/t/012_vm_consistency.pl b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
new file mode 100644
index 00000000000..7b9c577d19a
--- /dev/null
+++ b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
@@ -0,0 +1,205 @@
+# Copyright (c) 2021-2026, PostgreSQL Global Development Group
+#
+# Test that heap operations clearing visibility map bits (INSERT, UPDATE,
+# DELETE, SELECT FOR UPDATE, COPY) correctly register visibility map buffers,
+# since incremental backups rely on the WAL summarizer, which only tracks
+# registered buffers.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir_short();
+my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
+
+# Set up primary with WAL summarization enabled.
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(has_archiving => 1, allows_streaming => 1);
+$primary->append_conf('postgresql.conf', <<EOF);
+summarize_wal = on
+autovacuum = off
+EOF
+$primary->start;
+
+$primary->safe_psql('postgres', q{CREATE EXTENSION pg_visibility});
+
+my @tests = (
+ {
+ label => 'INSERT',
+ table => 'vm_insert_test',
+ setup => q{CREATE TABLE vm_insert_test (id int);
+ INSERT INTO vm_insert_test DEFAULT VALUES;},
+ modify => q{INSERT INTO vm_insert_test VALUES (1)},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'DELETE',
+ table => 'vm_delete_test',
+ setup => q{CREATE TABLE vm_delete_test (id int);
+ INSERT INTO vm_delete_test VALUES (1), (2);},
+ modify => q{DELETE FROM vm_delete_test WHERE id = 1},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'UPDATE',
+ table => 'vm_update_test',
+ # Include both same-page and cross-page updates.
+ setup => q{CREATE TABLE vm_update_test (id INT, val TEXT);
+ INSERT INTO vm_update_test VALUES (1, 'same page'), (2, 'cross page');
+ INSERT INTO vm_update_test SELECT i, repeat('a', 200)
+ FROM generate_series(3, 70) i;},
+ modify => q{UPDATE vm_update_test SET id = 0 WHERE id = 1;
+ UPDATE vm_update_test SET val = repeat('b', 8000) WHERE id = 2;},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'LOCK',
+ table => 'vm_lock_test',
+ setup => q{CREATE TABLE vm_lock_test (id int);
+ INSERT INTO vm_lock_test VALUES (1), (2);},
+ modify => q{SELECT * FROM vm_lock_test WHERE id = 1 FOR UPDATE},
+ visible_op => '==',
+ frozen_op => '<',
+ },
+ {
+ label => 'COPY',
+ table => 'vm_copy_test',
+ setup => q{CREATE TABLE vm_copy_test (id int);
+ INSERT INTO vm_copy_test DEFAULT VALUES;},
+ modify => q{COPY vm_copy_test FROM PROGRAM 'echo 42'},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+);
+
+sub get_vm_summary
+{
+ my ($node, $table) = @_;
+ my $result = $node->safe_psql('postgres',
+ "SELECT all_visible, all_frozen FROM pg_visibility_map_summary('$table')");
+ my @vals = split(/\|/, $result);
+ return @vals;
+}
+
+# Confirm VACUUM (FREEZE) set VM bits before testing whether later heap
+# modifications clear those bits and are captured by incremental backup.
+sub check_vacuumed_vm
+{
+ my ($node, $test) = @_;
+ my ($all_visible, $all_frozen) = get_vm_summary($node, $test->{table});
+
+ cmp_ok($all_visible, '>', 0,
+ "$test->{label} test: pages are all-visible after vacuum");
+ cmp_ok($all_frozen, '>', 0,
+ "$test->{label} test: pages are all-frozen after vacuum");
+
+ return ($all_visible, $all_frozen);
+}
+
+# Check the VM bit counts after a heap modification against the post-vacuum
+# baseline. Most operations clear both bits; tuple locking clears all-frozen
+# without clearing all-visible.
+sub check_modified_vm
+{
+ my ($node, $test) = @_;
+ my ($post_visible, $post_frozen) = get_vm_summary($node, $test->{table});
+
+ cmp_ok($post_visible, $test->{visible_op}, $test->{pre_visible},
+ "$test->{label} test: all-visible state after modification");
+ cmp_ok($post_frozen, $test->{frozen_op}, $test->{pre_frozen},
+ "$test->{label} test: all-frozen state after modification");
+
+ return ($post_visible, $post_frozen);
+}
+
+# Verify the combined backup restored VM state exactly as it exists on the
+# primary, and ask pg_visibility to check that visible tuples are consistent.
+sub validate_restored_vm
+{
+ my ($restored, $test) = @_;
+
+ my ($primary_visible, $primary_frozen) =
+ get_vm_summary($primary, $test->{table});
+ my ($restored_visible, $restored_frozen) =
+ get_vm_summary($restored, $test->{table});
+
+ is($restored_visible, $primary_visible,
+ "$test->{label} test: restored all_visible count matches primary");
+ is($restored_frozen, $primary_frozen,
+ "$test->{label} test: restored all_frozen count matches primary");
+
+ my $corrupt_tids = $restored->safe_psql('postgres',
+ "SELECT count(*) FROM pg_check_visible('$test->{table}')");
+ is($corrupt_tids, '0',
+ "$test->{label} test: no VM corruption detected by pg_check_visible");
+}
+
+# Create and populate the tables, then vacuum freeze them to set the VM bits
+foreach my $test (@tests)
+{
+ $primary->safe_psql('postgres', $test->{setup});
+ $primary->safe_psql('postgres', "VACUUM (FREEZE) $test->{table}");
+ ($test->{pre_visible}, $test->{pre_frozen}) =
+ check_vacuumed_vm($primary, $test);
+}
+
+# Take a full backup
+my $full_name = 'full';
+my $full_path = $primary->backup_dir . "/$full_name";
+$primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $full_path,
+ '--checkpoint' => 'fast',
+ ],
+ 'full backup');
+
+# Modify the tables and check that the VM bits are as expected for that test
+# after the specified modification.
+foreach my $test (@tests)
+{
+ $primary->safe_psql('postgres', $test->{modify});
+ ($test->{post_visible}, $test->{post_frozen}) =
+ check_modified_vm($primary, $test);
+}
+
+# Take an incremental backup. This will have the changes made in the
+# modification step.
+my $incr_name = 'incr';
+my $incr_path = $primary->backup_dir . "/$incr_name";
+$primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $incr_path,
+ '--checkpoint' => 'fast',
+ '--incremental' => $full_path . '/backup_manifest',
+ ],
+ 'incremental backup');
+
+# Start a server from a combined backup composed of the incremental and full
+# backup.
+my $restored = PostgreSQL::Test::Cluster->new('restored');
+$restored->init_from_backup($primary, $incr_name,
+ combine_with_prior => [$full_name],
+ combine_mode => $mode);
+$restored->append_conf('postgresql.conf', <<EOF);
+autovacuum = off
+EOF
+$restored->start;
+$restored->safe_psql('postgres', q{CREATE EXTENSION IF NOT EXISTS pg_visibility});
+
+# Confirm that the restored server's visibility map matches the original server
+foreach my $test (@tests)
+{
+ validate_restored_vm($restored, $test);
+}
+
+$restored->stop;
+$primary->stop;
+
+done_testing();
--
2.43.0
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-06-24 00:01 Melanie Plageman <melanieplageman@gmail.com>
parent: Tomas Vondra <tomas@vondra.me>
0 siblings, 0 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-06-24 00:01 UTC (permalink / raw)
To: Tomas Vondra <tomas@vondra.me>; +Cc: Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
On Thu, May 7, 2026 at 8:54 AM Tomas Vondra <tomas@vondra.me> wrote:
>
> On 5/6/26 23:46, Melanie Plageman wrote:
>
> > Based on an LLM-assisted self-review of this code, I think there is a
> > pre-existing bug in heap_lock_updated_tuple_rec() where we don't reset
> > cleared_all_frozen and may have a stale value if we iterate more than
> > once (new tuple versions) or use the l4 goto label (retries for the
> > same tuple version). This could mean we unnecessarily clear all-frozen
> > on the standby (and unnecessarily register the VM buffer after my
> > patch). I couldn't come up with a repro though, so I didn't include a
> > patch to fix it, as I wanted to confirm that someone else also thought
> > this is a bug.
>
> Hmm, would this be a correctness or efficiency issue?
>
> I'm not very familiar with this code, so I can't say if there is an
> issue or not. Could you explain the sequence of steps leading to an
> issue? Maybe that assumes something that's not quite possible, or it
> might help constructing a reproducer.
>
> I don't quite see how could it be related to the "l4" goto label, when
> all the "goto l4" cases are before setting "cleared_all_frozen = true"
> (in any given iteration).
You're right that it couldn't happen with goto l4. However, it is
possible in the loop when walking an update chain, it can set
cleared_all_frozen to true, clear the page's all frozen bits and then
later, on the next iteration, clear it again (even though it's already
clear). It's only an efficiency issue in master, actually (as you
suggest). In my patch it would be a serious bug because it would try
to register the VM page and set an LSN on that VM page when it isn't
locked or marked dirty.
- Melanie
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-06-26 20:05 Robert Haas <robertmhaas@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
0 siblings, 1 reply; 57+ messages in thread
From: Robert Haas @ 2026-06-26 20:05 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Tue, Jun 23, 2026 at 7:55 PM Melanie Plageman
<melanieplageman@gmail.com> wrote:
> I've done this in attached v3. I've only included proposed changes to
> 18, as this ended up being a pretty invasive refactor that I'd like to
> get feedback on first.
I'm finding that these patches don't apply cleanly to the latest sources.
[robert.haas pgsql]$ git log -n1 --oneline
4df5fe3833a (HEAD -> master, origin/master, origin/HEAD) Make
crosstabview honor boolean/null display settings
[robert.haas pgsql]$ patch -p1 <
~/Downloads/v3-0001-Alias-WAL-block-references-for-some-record-types.patch
patching file 'src/backend/access/heap/heapam.c'
1 out of 10 hunks failed--saving rejects to
'src/backend/access/heap/heapam.c.rej'
patching file 'src/backend/access/heap/heapam_xlog.c'
1 out of 16 hunks failed--saving rejects to
'src/backend/access/heap/heapam_xlog.c.rej'
patching file 'src/include/access/heapam_xlog.h'
Apparently these patches are based on an old commit, or you have some
other local patches. Can you specify a commit to apply them to, and/or
rebase?
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-06-27 10:41 Andrey Borodin <x4mmm@yandex-team.ru>
parent: Robert Haas <robertmhaas@gmail.com>
0 siblings, 2 replies; 57+ messages in thread
From: Andrey Borodin @ 2026-06-27 10:41 UTC (permalink / raw)
To: Robert Haas <robertmhaas@gmail.com>; +Cc: Melanie Plageman <melanieplageman@gmail.com>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
> On 27 Jun 2026, at 01:05, Robert Haas <robertmhaas@gmail.com> wrote:
>
> Apparently these patches are based on an old commit, or you have some
> other local patches. Can you specify a commit to apply them to, and/or
> rebase?
v3 set applies cleanly on REL_18_STABLE. +1 on backpatching this. I've done some
mechanical checks on the 18 variant: the new 012_vm_consistency test is green
(and goes red without the fix), and minor-version WAL compatibility holds:
WAL replays both ways between patched and unpatched 18.4 (patched replays
old-format records via the fallback path; unpatched harmlessly ignores the added
VM block refs).
Did I get it right that we will not have this
+ * Backwards compatibility path. Previously, the VM buffers were
on v19?
Best regards, Andrey Borodin.
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-06-29 07:46 Andrey Borodin <x4mmm@yandex-team.ru>
parent: Andrey Borodin <x4mmm@yandex-team.ru>
1 sibling, 1 reply; 57+ messages in thread
From: Andrey Borodin @ 2026-06-29 07:46 UTC (permalink / raw)
To: Robert Haas <robertmhaas@gmail.com>; +Cc: Melanie Plageman <melanieplageman@gmail.com>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
> On 27 Jun 2026, at 15:41, Andrey Borodin <x4mmm@yandex-team.ru> wrote:
>
> I've done some
> mechanical checks on the 18 variant
A couple of minor things:
- The new t/012_vm_consistency.pl needs pg_visibility, but
src/bin/pg_combinebackup/Makefile doesn't declare
EXTRA_INSTALL=contrib/pg_visibility, so a standalone
"make -C src/bin/pg_combinebackup check" fails with "extension pg_visibility
is not available". (meson test and make check-world are fine, since they install
the whole tree) (for example see how src/bin/pg_amcheck pulls amcheck/pageinspect)
- A tiny wording nit: the comment on the retained visibilitymap_clear()
says it's kept "for backwards compatibility". Maybe spell out that it's kept for
ABI(prototype stability) on the back branches?
Best regards, Andrey Borodin.
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-06-29 14:42 Robert Haas <robertmhaas@gmail.com>
parent: Andrey Borodin <x4mmm@yandex-team.ru>
1 sibling, 1 reply; 57+ messages in thread
From: Robert Haas @ 2026-06-29 14:42 UTC (permalink / raw)
To: Andrey Borodin <x4mmm@yandex-team.ru>; +Cc: Melanie Plageman <melanieplageman@gmail.com>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Sat, Jun 27, 2026 at 6:41 AM Andrey Borodin <x4mmm@yandex-team.ru> wrote:
> v3 set applies cleanly on REL_18_STABLE.
Oh, I didn't realize this was based on the back-branch code. Thanks.
I have reviewed 0001 and 0002 and in general I think this looks good.
There is possibly room for some concern about performance regressions
with 0002 since, as you point out in some of the added comments, we
will now keep the VM buffer locked for longer. It might be worth
benchmarking that to see how bad it looks, but I don't know that we
really have any choice. Correctness bugs are bad.
One problem is that I think the newly-registered VM blocks will get
fed to heap_mask() if wal_consistency_checking is turned on, but it's
really only meant to work with heap pages.
Other comments follow.
0001. I personally don't find this to be an improvement. Even it is, I
feel like a back-patched commit is not a good place to invent a new
convention for identifying block reference indexes. However, it's
ultimately a matter of style, so if I lose this particular argument,
c'est la vie. I just feel like if we introduce this for every WAL
record, it's going to be a lot of new #defines that are going to be
long and clunky, and if we don't, then we have a weird mix of styles.
0002. In heap_insert, the local variable called 'all_visible_cleared'
seems like it needs a bit of work. You'd expect such a variable to be
set at the point where we clear the all-visible bit, but it's actually
set before that, when we lock the buffer. Later, you write in the
comments "We locked vmbuffer if all_visible_cleared was true
regardless of whether or not we ended up modifying it," which
similarly implies that the variable is now misnamed. However,
PageClearAllVisible() is called if and only if all_visible_cleared is
true, so that comment doesn't seem to be entirely accurate. My first
instinct is to rename the variable to clear_all_visible and delete the
comment, but there might be something better.
Looks like heap_delete has the same issue, and heap_update has a
similar issue multiplied by two (all_visible_cleared vs.
all_visible_cleared_new).
In heap_update, the comment /* clear PD_ALL_VISIBLE flags, reset all
visibilitymap bits */ has been lost, and the code has been reordered
slightly in that area. This is fine, but keeping the existing comment
and/or code ordering could be worth considering.
In heap_lock_tuple, I think it might be a good idea to set
unlock_vmbuffer = false when we release the lock, and then just after
the out_unlocked: label, Assert(!unlock_vmbuffer). That way, if
someone adds an improvident goto in a future patch, it has a chance of
tripping an Assert.
I think the comment for heap_xlog_vm_clear could be better: instead of
saying that it "handles" something, say what it actually does.
Likewise "use it for redo" inside the function is a little imprecise.
If I'm understanding the purpose of this machinery correctly, the
point here is that any FPIs included for the VM block references need
to be applied, and otherwise (if there's no FPI or the block reference
is not included at all) we need to just clear the relevant bit. That
seems correct, but a little more work on the comments would make it
clearer to future readers, I feel.
In heap_xlog_multi_insert, you delete the comment "The visibility map
may need to be fixed even if the heap page is already up-to-date," and
replace it with a completely new comment that seems to be addresing
someone different concerns, but the identical comments in other
functions in the file are retained. Consider adding to the comment
instead of replacing it. Also, I am not sure I understand the point of
the new comment. Perhaps try to make it clearer.
I find the phrasing of the comment addition at the top of
visibilitymap_clear to be a little harder to understand than
necessary. Maybe: "Most callers should use visibilitymap_clear_locked
rather than this function. It is usually necessary to register the VM
buffer in the WAL record, and this necessitates holding the lock for
longer than it is held here. However, we retain this function for
backward compatibility, and for a few callers that don't have this
requirement."
The redo routines ignore the return value of
visibilitymap_clear_locked, which means they set the page LSN even if
it isn't modified. Apparently there is other precedent for that
treatment, but maybe it's best avoided anyway.
I think that the commit messages could use some polishing. For 0001,
if we keep it, I'd suggest that the subject line should use language
like "introduce macros for" instead of "alias" because nobody's going
to know what "alias" means without reading the commit itself. For
0002, I'd suggest that the subject line should be something like "Fix
heap WAL records to register VM buffers when clearing VM bits". I
think your goals here should be (1) to maximize the chance that
somebody can understand what the commit does if they just read the
one-line summary, or at most the whole commit message, and (2) to
minimize the amount of work it will take to turn your commit message
into a release note entry. I'm not saying your existing commit
messages are bad, but they read a bit as if they were written to
summarize work that the reader has already seen.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-10 19:06 Melanie Plageman <melanieplageman@gmail.com>
parent: Robert Haas <robertmhaas@gmail.com>
0 siblings, 2 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-07-10 19:06 UTC (permalink / raw)
To: Robert Haas <robertmhaas@gmail.com>; +Cc: Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Mon, Jun 29, 2026 at 10:43 AM Robert Haas <robertmhaas@gmail.com> wrote:
>
> I have reviewed 0001 and 0002 and in general I think this looks good.
> There is possibly room for some concern about performance regressions
> with 0002 since, as you point out in some of the added comments, we
> will now keep the VM buffer locked for longer. It might be worth
> benchmarking that to see how bad it looks, but I don't know that we
> really have any choice. Correctness bugs are bad.
Thanks for the review!
As for performance regressions, I concocted what I felt was the worst
case (repro script pasted at the bottom of the email) which is a table
with all the pages covered by a single VM page and 32 clients each
trying to clear VM bits from different heap pages. It's not a perfect
benchmark because the only way to get the pages to keep being set
all-visible between updates was to run vacuum separately in a loop and
that wouldn't produce the same pattern of vacuums from run-to-run. I
didn't see any performance degradation for this with mostly default
settings. If I turn synchronous_commit off, I see a 10-15% TPS
performance degradation. This seems due to the extra time holding the
VM buffer lock. If you sample wait events, you see a big increase in
BufferContent lock wait events with the patch. As you mentioned, I
don't think there is anything we can do about it, though, and it
doesn't seem to be the bottleneck in the common configuration.
> One problem is that I think the newly-registered VM blocks will get
> fed to heap_mask() if wal_consistency_checking is turned on, but it's
> really only meant to work with heap pages.
visibilitymap_set() registers VM blocks already and those go through
wal consistency checking like all other HEAP2 records. It isn't great,
but it is basically harmless because heap_mask() loop conditions will
cause it to do nothing anyway. We could perhaps do something better
with non-heap pages in wal consistency checking, but we certainly
shouldn't do that on back branches.
> 0001. I personally don't find this to be an improvement. Even it is, I
> feel like a back-patched commit is not a good place to invent a new
> convention for identifying block reference indexes. However, it's
> ultimately a matter of style, so if I lose this particular argument,
> c'est la vie. I just feel like if we introduce this for every WAL
> record, it's going to be a lot of new #defines that are going to be
> long and clunky, and if we don't, then we have a weird mix of styles.
I agree with you that this is less than ideal on back branches and not
great that we don't have some way of enforcing people continuing to do
it even when done on master (leading to inconsistent usage).
However, I find registering VM buffers untenably confusing without
naming the various block IDs registered. Especially because we don't
replay them in the same order they were registered in. And for
heap_update(), we register up to 4 blocks (after the fix) and the
order is significant because we may be using one VM block for two heap
blocks. I had at some point suggested only macro-izing the update
block refs, but another reviewer preferred aliasing all of them, as it
isn't immediately clear what that block ID is with even two blocks
registered.
> 0002. In heap_insert, the local variable called 'all_visible_cleared'
> seems like it needs a bit of work. You'd expect such a variable to be
> set at the point where we clear the all-visible bit, but it's actually
> set before that, when we lock the buffer. Later, you write in the
> comments "We locked vmbuffer if all_visible_cleared was true
> regardless of whether or not we ended up modifying it," which
> similarly implies that the variable is now misnamed. However,
> PageClearAllVisible() is called if and only if all_visible_cleared is
> true, so that comment doesn't seem to be entirely accurate. My first
> instinct is to rename the variable to clear_all_visible and delete the
> comment, but there might be something better.
I had originally wanted to keep the name all_visible_cleared since it
was an existing variable, but you're right that that is now confusing.
I've changed it as you suggested. I think the comment is clarifying
because we don't have a separate unlock_vmbuffer variable here.
> In heap_update, the comment /* clear PD_ALL_VISIBLE flags, reset all
> visibilitymap bits */ has been lost, and the code has been reordered
> slightly in that area. This is fine, but keeping the existing comment
> and/or code ordering could be worth considering.
>
> In heap_lock_tuple, I think it might be a good idea to set
> unlock_vmbuffer = false when we release the lock, and then just after
> the out_unlocked: label, Assert(!unlock_vmbuffer). That way, if
> someone adds an improvident goto in a future patch, it has a chance of
> tripping an Assert.
I've done these two things.
> I think the comment for heap_xlog_vm_clear could be better: instead of
> saying that it "handles" something, say what it actually does.
> Likewise "use it for redo" inside the function is a little imprecise.
> If I'm understanding the purpose of this machinery correctly, the
> point here is that any FPIs included for the VM block references need
> to be applied, and otherwise (if there's no FPI or the block reference
> is not included at all) we need to just clear the relevant bit. That
> seems correct, but a little more work on the comments would make it
> clearer to future readers, I feel.
I think I've improved this.
> In heap_xlog_multi_insert, you delete the comment "The visibility map
> may need to be fixed even if the heap page is already up-to-date," and
> replace it with a completely new comment that seems to be addresing
> someone different concerns, but the identical comments in other
> functions in the file are retained. Consider adding to the comment
> instead of replacing it. Also, I am not sure I understand the point of
> the new comment. Perhaps try to make it clearer.
I didn't understand the existing comment or even what it was trying to
say, which is why I deleted it. I've restored it now and tried to
clarify my additional point.
> I find the phrasing of the comment addition at the top of
> visibilitymap_clear to be a little harder to understand than
> necessary. Maybe: "Most callers should use visibilitymap_clear_locked
> rather than this function. It is usually necessary to register the VM
> buffer in the WAL record, and this necessitates holding the lock for
> longer than it is held here. However, we retain this function for
> backward compatibility, and for a few callers that don't have this
> requirement."
I adopted this phrasing.
> The redo routines ignore the return value of
> visibilitymap_clear_locked, which means they set the page LSN even if
> it isn't modified. Apparently there is other precedent for that
> treatment, but maybe it's best avoided anyway.
Done.
> I think that the commit messages could use some polishing. For 0001,
> if we keep it, I'd suggest that the subject line should use language
> like "introduce macros for" instead of "alias" because nobody's going
> to know what "alias" means without reading the commit itself. For
> 0002, I'd suggest that the subject line should be something like "Fix
> heap WAL records to register VM buffers when clearing VM bits". I
> think your goals here should be (1) to maximize the chance that
> somebody can understand what the commit does if they just read the
> one-line summary, or at most the whole commit message, and (2) to
> minimize the amount of work it will take to turn your commit message
> into a release note entry. I'm not saying your existing commit
> messages are bad, but they read a bit as if they were written to
> summarize work that the reader has already seen.
I've tried to clarify my commit messages.
Your original review didn't mention 0003/0004 (the test commits). I
was hoping to get a review especially on 0003.
For 0004, I feel relatively confident in it, but I am wondering if the
extra validation I added to verify that the updates happen either on a
single page or across pages is overkill or not. It will make sure the
test continues to cover this case, but, it is quite a few extra lines
of test code.
For 0003, I am much less confident. I've never written test
infrastructure in Perl before. I used AI to help me write it, so I
could really use review from someone experienced with adding TAP test
utility functions. A few thoughts on it/questions:
_run_pg_combinebackup_with_debug_log's error handling is basically
copied from system_or_bail -- is that too duplicative? Also, I'm not
sure if the END code block is done correctly or not.
I think the overall behavior seems okay -- make a temp directory for
the logs from each pg_combinebackup run. Store the paths to those
directories in an array. If pg_combinebackup invocation itself fails,
print the verbose logs right away. Otherwise, at the end of the whole
test run, if any tests failed, print all the verbose logs. If no tests
failed, unlink the test files. It doesn't actually remove the temp
dirs though -- I think those are cleaned up in the general test
cleanup.
One annoying thing is we save the logs at the time of the
pg_combinebackup run, so we don't know if the assertions following it
pass or succeed. That means if any of the assertions fail we will
print all logs. This may be unavoidable, as we wouldn't even be sure
which logs to attribute to which assertions.
Also combinebackup_debug_logs_printed hash set is a little
over-complicated -- it was added because we could print the same log
output twice if pg_combinebackup itself failed. Maybe there is a less
complicated way to handle this? I don't think the END block is
guaranteed to run depending on how pg_combinebackup fails, so I don't
know that we can be sure the logs print without doing it right when
pg_combinebackup fails.
- Melanie
repro (set sync commit off to see the regression):
psql -c "ALTER SYSTEM SET shared_buffers = '2GB';" \
-c "ALTER SYSTEM SET max_wal_size = '150 GB';" \
-c "ALTER SYSTEM SET maintenance_work_mem = '1GB';" \
-c "alter system set synchronous_commit = on"
pg_ctl -l $PGLOG restart
psql -f- <<EOF
DROP TABLE IF EXISTS foo;
CREATE TABLE foo (
cid int not null, -- which client owns the row
k int not null, -- key within that client's partition
filler text not null
) WITH (autovacuum_enabled = off);
ALTER TABLE foo ALTER COLUMN filler SET STORAGE PLAIN;
INSERT INTO foo
SELECT (g-1) % 16, (g-1) / 16, repeat('x', 4200)
FROM generate_series(1, 32000) g;
CREATE UNIQUE INDEX foo_cid_k ON foo (cid, k); -- avoid seq scans
VACUUM (FREEZE, DISABLE_PAGE_SKIPPING) foo;
EOF
while :; do psql -c "VACUUM (FREEZE) foo;"; sleep 0.05; done &
pgbench -n -c 32 -j 32 -t 10000 -D rpc=2000 \
-f- <<EOF
\set k random(0, :rpc - 1)
UPDATE foo SET filler = filler WHERE cid = :client_id AND k = :k;
EOF
kill %1
Attachments: download all (4 files, .tar.gz)
v4-0001-Introduce-macros-for-WAL-block-reference-IDs-of-s.patch (text/x-patch, 15.8K) download
From 8bacdc59c088c19b60c74fc3d5dcb815fe7ff14b Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Mon, 4 May 2026 10:41:13 -0400
Subject: [PATCH v4 1/4] Introduce macros for WAL block reference IDs of some
heap record types
When registering a buffer with the WAL machinery, the caller assigns it a
block reference ID, and replay must read each block back by that same ID.
Today these IDs are bare integers assigned by convention (0, 1, 2, ...),
which is easy to follow when a record registers a single block, or when the
blocks are handled during replay in their registration order.
An upcoming bug fix registers up to two visibility map blocks in addition
to the heap block(s) when clearing the VM, and these are not handled during
replay in a straightforward 1:1, in-registration-order fashion. Relying on
bare integers for the block IDs in that case is error-prone.
Introduce macros naming the block reference IDs for the heap record types
that the upcoming commit extends to register visibility map blocks, so the
registration and replay sites refer to the same block by a meaningful name.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Robert Haas <robertmhaas@gmail.com>
Discussion: https://postgr.es/m/66mqpfyti3qhfttcsv6r2lbvqqd32rrmpn6i47ovrsnvguts46%40gou54xc>
Backpatch through: 17
---
src/backend/access/heap/heapam.c | 43 +++++++++++---------
src/backend/access/heap/heapam_xlog.c | 56 +++++++++++++++++----------
src/include/access/heapam_xlog.h | 28 ++++++++++----
3 files changed, 82 insertions(+), 45 deletions(-)
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 6203e3d7f8d..96c25197788 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2218,10 +2218,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
* write the whole page to the xlog, we don't need to store
* xl_heap_header in the xlog.
*/
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
- XLogRegisterBufData(0, &xlhdr, SizeOfHeapHeader);
+ XLogRegisterBuffer(HEAP_INSERT_BLKREF_HEAP, buffer,
+ REGBUF_STANDARD | bufflags);
+ XLogRegisterBufData(HEAP_INSERT_BLKREF_HEAP, &xlhdr,
+ SizeOfHeapHeader);
/* PG73FORMAT: write bitmap [+ padding] [+ oid] + data */
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_INSERT_BLKREF_HEAP,
(char *) heaptup->t_data + SizeofHeapTupleHeader,
heaptup->t_len - SizeofHeapTupleHeader);
@@ -2619,9 +2621,11 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogBeginInsert();
XLogRegisterData(xlrec, tupledata - scratch.data);
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
+ XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_HEAP, buffer,
+ REGBUF_STANDARD | bufflags);
- XLogRegisterBufData(0, tupledata, totaldatalen);
+ XLogRegisterBufData(HEAP_MULTI_INSERT_BLKREF_HEAP, tupledata,
+ totaldatalen);
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
@@ -3112,7 +3116,7 @@ l1:
XLogBeginInsert();
XLogRegisterData(&xlrec, SizeOfHeapDelete);
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_DELETE_BLKREF_HEAP, buffer, REGBUF_STANDARD);
/*
* Log replica identity of the deleted tuple if there is one
@@ -3883,7 +3887,7 @@ l2:
XLogRecPtr recptr;
XLogBeginInsert();
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buffer, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&oldtup.t_self);
xlrec.xmax = xmax_lock_old_tuple;
@@ -5219,7 +5223,7 @@ failed:
XLogRecPtr recptr;
XLogBeginInsert();
- XLogRegisterBuffer(0, *buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, *buffer, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&tuple->t_self);
xlrec.xmax = xid;
@@ -5971,7 +5975,7 @@ l4:
Page page = BufferGetPage(buf);
XLogBeginInsert();
- XLogRegisterBuffer(0, buf, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buf, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&mytup.t_self);
xlrec.xmax = new_xmax;
@@ -8972,9 +8976,9 @@ log_heap_update(Relation reln, Buffer oldbuf,
if (need_tuple_data)
bufflags |= REGBUF_KEEP_DATA;
- XLogRegisterBuffer(0, newbuf, bufflags);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_HEAP_NEW, newbuf, bufflags);
if (oldbuf != newbuf)
- XLogRegisterBuffer(1, oldbuf, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_HEAP_OLD, oldbuf, REGBUF_STANDARD);
XLogRegisterData(&xlrec, SizeOfHeapUpdate);
@@ -8987,15 +8991,18 @@ log_heap_update(Relation reln, Buffer oldbuf,
{
prefix_suffix[0] = prefixlen;
prefix_suffix[1] = suffixlen;
- XLogRegisterBufData(0, &prefix_suffix, sizeof(uint16) * 2);
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW, &prefix_suffix,
+ sizeof(uint16) * 2);
}
else if (prefixlen > 0)
{
- XLogRegisterBufData(0, &prefixlen, sizeof(uint16));
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW, &prefixlen,
+ sizeof(uint16));
}
else
{
- XLogRegisterBufData(0, &suffixlen, sizeof(uint16));
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW, &suffixlen,
+ sizeof(uint16));
}
}
@@ -9009,10 +9016,10 @@ log_heap_update(Relation reln, Buffer oldbuf,
*
* The 'data' doesn't include the common prefix or suffix.
*/
- XLogRegisterBufData(0, &xlhdr, SizeOfHeapHeader);
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW, &xlhdr, SizeOfHeapHeader);
if (prefixlen == 0)
{
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW,
(char *) newtup->t_data + SizeofHeapTupleHeader,
newtup->t_len - SizeofHeapTupleHeader - suffixlen);
}
@@ -9025,13 +9032,13 @@ log_heap_update(Relation reln, Buffer oldbuf,
/* bitmap [+ padding] [+ oid] */
if (newtup->t_data->t_hoff - SizeofHeapTupleHeader > 0)
{
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW,
(char *) newtup->t_data + SizeofHeapTupleHeader,
newtup->t_data->t_hoff - SizeofHeapTupleHeader);
}
/* data after common prefix */
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW,
(char *) newtup->t_data + newtup->t_data->t_hoff + prefixlen,
newtup->t_len - newtup->t_data->t_hoff - prefixlen - suffixlen);
}
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index eb4bd3d6ae3..87b5c44fd46 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -350,7 +350,8 @@ heap_xlog_delete(XLogReaderState *record)
RelFileLocator target_locator;
ItemPointerData target_tid;
- XLogRecGetBlockTag(record, 0, &target_locator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_DELETE_BLKREF_HEAP, &target_locator, NULL,
+ &blkno);
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
@@ -369,7 +370,8 @@ heap_xlog_delete(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_DELETE_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = BufferGetPage(buffer);
@@ -434,7 +436,8 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerData target_tid;
XLogRedoAction action;
- XLogRecGetBlockTag(record, 0, &target_locator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_INSERT_BLKREF_HEAP, &target_locator, NULL,
+ &blkno);
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
@@ -462,13 +465,14 @@ heap_xlog_insert(XLogReaderState *record)
*/
if (XLogRecGetInfo(record) & XLOG_HEAP_INIT_PAGE)
{
- buffer = XLogInitBufferForRedo(record, 0);
+ buffer = XLogInitBufferForRedo(record, HEAP_INSERT_BLKREF_HEAP);
page = BufferGetPage(buffer);
PageInit(page, BufferGetPageSize(buffer), 0);
action = BLK_NEEDS_REDO;
}
else
- action = XLogReadBufferForRedo(record, 0, &buffer);
+ action = XLogReadBufferForRedo(record, HEAP_INSERT_BLKREF_HEAP,
+ &buffer);
if (action == BLK_NEEDS_REDO)
{
Size datalen;
@@ -479,7 +483,7 @@ heap_xlog_insert(XLogReaderState *record)
if (PageGetMaxOffsetNumber(page) + 1 < xlrec->offnum)
elog(PANIC, "invalid max offset number");
- data = XLogRecGetBlockData(record, 0, &datalen);
+ data = XLogRecGetBlockData(record, HEAP_INSERT_BLKREF_HEAP, &datalen);
newlen = datalen - SizeOfHeapHeader;
Assert(datalen > SizeOfHeapHeader && newlen <= MaxHeapTupleSize);
@@ -559,7 +563,8 @@ heap_xlog_multi_insert(XLogReaderState *record)
*/
xlrec = (xl_heap_multi_insert *) XLogRecGetData(record);
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_MULTI_INSERT_BLKREF_HEAP, &rlocator, NULL,
+ &blkno);
/* check that the mutually exclusive flags are not both set */
Assert(!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
@@ -582,13 +587,14 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (isinit)
{
- buffer = XLogInitBufferForRedo(record, 0);
+ buffer = XLogInitBufferForRedo(record, HEAP_MULTI_INSERT_BLKREF_HEAP);
page = BufferGetPage(buffer);
PageInit(page, BufferGetPageSize(buffer), 0);
action = BLK_NEEDS_REDO;
}
else
- action = XLogReadBufferForRedo(record, 0, &buffer);
+ action = XLogReadBufferForRedo(record, HEAP_MULTI_INSERT_BLKREF_HEAP,
+ &buffer);
if (action == BLK_NEEDS_REDO)
{
char *tupdata;
@@ -596,7 +602,8 @@ heap_xlog_multi_insert(XLogReaderState *record)
Size len;
/* Tuples are stored as block data */
- tupdata = XLogRecGetBlockData(record, 0, &len);
+ tupdata = XLogRecGetBlockData(record, HEAP_MULTI_INSERT_BLKREF_HEAP,
+ &len);
endptr = tupdata + len;
page = (Page) BufferGetPage(buffer);
@@ -713,8 +720,10 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
oldtup.t_data = NULL;
oldtup.t_len = 0;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &newblk);
- if (XLogRecGetBlockTagExtended(record, 1, NULL, NULL, &oldblk, NULL))
+ XLogRecGetBlockTag(record, HEAP_UPDATE_BLKREF_HEAP_NEW, &rlocator, NULL,
+ &newblk);
+ if (XLogRecGetBlockTagExtended(record, HEAP_UPDATE_BLKREF_HEAP_OLD, NULL, NULL,
+ &oldblk, NULL))
{
/* HOT updates are never done across pages */
Assert(!hot_update);
@@ -750,7 +759,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
*/
/* Deal with old tuple version */
- oldaction = XLogReadBufferForRedo(record, (oldblk == newblk) ? 0 : 1,
+ oldaction = XLogReadBufferForRedo(record, (oldblk == newblk) ?
+ HEAP_UPDATE_BLKREF_HEAP_NEW : HEAP_UPDATE_BLKREF_HEAP_OLD,
&obuffer);
if (oldaction == BLK_NEEDS_REDO)
{
@@ -800,13 +810,14 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
}
else if (XLogRecGetInfo(record) & XLOG_HEAP_INIT_PAGE)
{
- nbuffer = XLogInitBufferForRedo(record, 0);
+ nbuffer = XLogInitBufferForRedo(record, HEAP_UPDATE_BLKREF_HEAP_NEW);
page = (Page) BufferGetPage(nbuffer);
PageInit(page, BufferGetPageSize(nbuffer), 0);
newaction = BLK_NEEDS_REDO;
}
else
- newaction = XLogReadBufferForRedo(record, 0, &nbuffer);
+ newaction = XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_HEAP_NEW,
+ &nbuffer);
/*
* The visibility map may need to be fixed even if the heap page is
@@ -831,7 +842,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Size datalen;
Size tuplen;
- recdata = XLogRecGetBlockData(record, 0, &datalen);
+ recdata = XLogRecGetBlockData(record, HEAP_UPDATE_BLKREF_HEAP_NEW,
+ &datalen);
recdata_end = recdata + datalen;
page = BufferGetPage(nbuffer);
@@ -1015,7 +1027,8 @@ heap_xlog_lock(XLogReaderState *record)
BlockNumber block;
Relation reln;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
+ XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
+ &block);
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
@@ -1025,7 +1038,8 @@ heap_xlog_lock(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = (Page) BufferGetPage(buffer);
@@ -1091,7 +1105,8 @@ heap_xlog_lock_updated(XLogReaderState *record)
BlockNumber block;
Relation reln;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
+ XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
+ &block);
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
@@ -1101,7 +1116,8 @@ heap_xlog_lock_updated(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = BufferGetPage(buffer);
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 277df6b3cf0..92744c53899 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -110,6 +110,8 @@
(XLH_DELETE_CONTAINS_OLD_TUPLE | XLH_DELETE_CONTAINS_OLD_KEY)
/* This is what we need to know about delete */
+#define HEAP_DELETE_BLKREF_HEAP 0
+
typedef struct xl_heap_delete
{
TransactionId xmax; /* xmax of the deleted tuple */
@@ -157,12 +159,14 @@ typedef struct xl_heap_header
#define SizeOfHeapHeader (offsetof(xl_heap_header, t_hoff) + sizeof(uint8))
/* This is what we need to know about insert */
+#define HEAP_INSERT_BLKREF_HEAP 0
+
typedef struct xl_heap_insert
{
OffsetNumber offnum; /* inserted tuple's offset */
uint8 flags;
- /* xl_heap_header & TUPLE DATA in backup block 0 */
+ /* xl_heap_header & TUPLE DATA in HEAP_INSERT_BLKREF_HEAP */
} xl_heap_insert;
#define SizeOfHeapInsert (offsetof(xl_heap_insert, flags) + sizeof(uint8))
@@ -173,10 +177,13 @@ typedef struct xl_heap_insert
* The main data of the record consists of this xl_heap_multi_insert header.
* 'offsets' array is omitted if the whole page is reinitialized
* (XLOG_HEAP_INIT_PAGE).
- *
- * In block 0's data portion, there is an xl_multi_insert_tuple struct,
- * followed by the tuple data for each tuple. There is padding to align
- * each xl_multi_insert_tuple struct.
+ */
+#define HEAP_MULTI_INSERT_BLKREF_HEAP 0
+
+/*
+ * In HEAP_MULTI_INSERT_BLKREF_HEAP's data portion, there is an
+ * xl_multi_insert_tuple struct, followed by the tuple data for each tuple.
+ * There is padding to align each xl_multi_insert_tuple struct.
*/
typedef struct xl_heap_multi_insert
{
@@ -201,7 +208,7 @@ typedef struct xl_multi_insert_tuple
/*
* This is what we need to know about update|hot_update
*
- * Backup blk 0: new page
+ * HEAP_UPDATE_BLKREF_HEAP_NEW: new page
*
* If XLH_UPDATE_PREFIX_FROM_OLD or XLH_UPDATE_SUFFIX_FROM_OLD flags are set,
* the prefix and/or suffix come first, as one or two uint16s.
@@ -213,8 +220,13 @@ typedef struct xl_multi_insert_tuple
* If XLH_UPDATE_CONTAINS_NEW_TUPLE flag is given, the tuple data is
* included even if a full-page image was taken.
*
- * Backup blk 1: old page, if different. (no data, just a reference to the blk)
+ * HEAP_UPDATE_BLKREF_HEAP_OLD: old page, if different. (no data, just a reference
+ * to the block)
*/
+
+#define HEAP_UPDATE_BLKREF_HEAP_NEW 0
+#define HEAP_UPDATE_BLKREF_HEAP_OLD 1
+
typedef struct xl_heap_update
{
TransactionId old_xmax; /* xmax of the old tuple */
@@ -393,6 +405,8 @@ typedef struct xlhp_prune_items
#define XLH_LOCK_ALL_FROZEN_CLEARED 0x01
/* This is what we need to know about lock */
+#define HEAP_LOCK_BLKREF_HEAP 0
+
typedef struct xl_heap_lock
{
TransactionId xmax; /* might be a MultiXactId */
--
2.47.3
v4-0002-Fix-VM-clear-WAL-logging-by-registering-VM-blocks.patch (text/x-patch, 45.6K) download
From 8ddc25481951f9115fecd6c8880f3bf14657e5eb Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 12:53:49 -0400
Subject: [PATCH v4 2/4] Fix VM clear WAL logging by registering VM blocks
Heap WAL records that clear bits on the visibility map (like inserts and
deletes) did not register the visibility map blocks they modified.
Because the WAL summarizer only records registered blocks, an
incremental backup taken over such operations would omit the changed VM
pages. On restore, the VM would retain stale all-visible/all-frozen
bits, which can cause wrong results from index-only scans and incorrect
relfrozenxid advancement due to vacuum page skipping.
Not registering the VM buffer also meant we never emitted FPIs of VM
pages when clearing bits. A torn VM page won't raise an error because
the VM is read with ZERO_ON_ERROR; with checksums on, it would be
detected and zeroed, but with checksums off, it is accepted as-is and
can lead to data corruption.
Fix this by registering the VM buffer in the WAL record when clearing VM
bits. The VM buffer must now be locked throughout the critical section
that modifies the VM and heap pages and emits the WAL record. This can
slow down operations that clear the VM, since the VM lock is held longer
and VM FPIs may be emitted, but it is required for correctness.
Note that this fix does not repair existing incremental backups.
Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Andres Freund <andres@anarazel.de>
Reviewed-by: Robert Haas <robertmhaas@gmail.com>
Reviewed-by: Andrey Borodin <x4mmm@yandex-team.ru>
Discussion: https://postgr.es/m/flat/CAAKRu_bn%2Be7F4yPFBgFbnP%2BsyJRKyNK092bjD2LKvZW7O4Svag>
Backpatch-through: 17
---
contrib/pg_surgery/heap_surgery.c | 40 ++-
src/backend/access/heap/heapam.c | 383 ++++++++++++++++++++----
src/backend/access/heap/heapam_xlog.c | 239 ++++++++++-----
src/backend/access/heap/visibilitymap.c | 34 ++-
src/bin/pg_walsummary/t/002_blocks.pl | 7 +-
src/include/access/heapam_xlog.h | 17 +-
src/include/access/visibilitymap.h | 2 +
7 files changed, 574 insertions(+), 148 deletions(-)
diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c
index 602aca66c60..6a38ac577c6 100644
--- a/contrib/pg_surgery/heap_surgery.c
+++ b/contrib/pg_surgery/heap_surgery.c
@@ -17,6 +17,7 @@
#include "access/visibilitymap.h"
#include "access/xloginsert.h"
#include "catalog/pg_am_d.h"
+#include "catalog/pg_control.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
#include "utils/acl.h"
@@ -146,6 +147,7 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
{
Buffer buf;
Buffer vmbuf = InvalidBuffer;
+ bool unlock_vmbuf = false;
Page page;
BlockNumber blkno;
OffsetNumber curoff;
@@ -233,11 +235,15 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
}
/*
- * Before entering the critical section, pin the visibility map page
- * if it appears to be necessary.
+ * Before entering the critical section, pin and lock the visibility
+ * map page if it appears to be necessary.
*/
if (heap_force_opt == HEAP_FORCE_KILL && PageIsAllVisible(page))
+ {
visibilitymap_pin(rel, blkno, &vmbuf);
+ LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuf = true;
+ }
/* No ereport(ERROR) from here until all the changes are logged. */
START_CRIT_SECTION();
@@ -266,10 +272,11 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
*/
if (PageIsAllVisible(page))
{
+ if (visibilitymap_clear_locked(rel, blkno, vmbuf,
+ VISIBILITYMAP_VALID_BITS))
+ did_modify_vm = true;
+
PageClearAllVisible(page);
- visibilitymap_clear(rel, blkno, vmbuf,
- VISIBILITYMAP_VALID_BITS);
- did_modify_vm = true;
}
}
else
@@ -320,18 +327,29 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
/* XLOG stuff */
if (RelationNeedsWAL(rel))
- log_newpage_buffer(buf, true);
+ {
+ XLogRecPtr recptr;
+
+ XLogBeginInsert();
+ XLogRegisterBuffer(0, buf, REGBUF_STANDARD | REGBUF_FORCE_IMAGE);
+ /* Include the VM page if it was modified. */
+ if (did_modify_vm)
+ XLogRegisterBuffer(1, vmbuf, REGBUF_FORCE_IMAGE);
+ recptr = XLogInsert(RM_XLOG_ID, XLOG_FPI);
+ if (did_modify_vm)
+ PageSetLSN(BufferGetPage(vmbuf), recptr);
+ PageSetLSN(BufferGetPage(buf), recptr);
+ }
}
- /* WAL log the VM page if it was modified. */
- if (did_modify_vm && RelationNeedsWAL(rel))
- log_newpage_buffer(vmbuf, false);
-
END_CRIT_SECTION();
UnlockReleaseBuffer(buf);
- if (vmbuf != InvalidBuffer)
+ if (unlock_vmbuf)
+ LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
+
+ if (BufferIsValid(vmbuf))
ReleaseBuffer(vmbuf);
/* Update the current_start_ptr before moving to the next page. */
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 96c25197788..c3f611a42ed 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -58,7 +58,8 @@
static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup,
TransactionId xid, CommandId cid, int options);
static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup,
+ Buffer vmbuffer_old, Buffer newbuf,
+ Buffer vmbuffer_new, HeapTuple oldtup,
HeapTuple newtup, HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared);
#ifdef USE_ASSERT_CHECKING
@@ -2083,8 +2084,10 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
- bool all_visible_cleared = false;
+ bool clear_all_visible = false;
+ bool vmbuffer_modified = false;
/* Cheap, simplistic check that the tuple matches the rel's rowtype. */
Assert(HeapTupleHeaderGetNatts(tup->t_data) <=
@@ -2108,6 +2111,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
InvalidBuffer, options, bistate,
&vmbuffer, NULL,
0);
+ page = BufferGetPage(buffer);
/*
* We're about to do the actual insert -- but check for conflict first, to
@@ -2126,19 +2130,28 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
*/
CheckForSerializableConflictIn(relation, NULL, InvalidBlockNumber);
+ /* Lock the vmbuffer before the critical section */
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ clear_all_visible = true;
+ }
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ if (clear_all_visible)
{
- all_visible_cleared = true;
- PageClearAllVisible(BufferGetPage(buffer));
- visibilitymap_clear(relation,
- ItemPointerGetBlockNumber(&(heaptup->t_self)),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ /* It's possible the VM bits were already clear */
+ if (visibilitymap_clear_locked(relation,
+ ItemPointerGetBlockNumber(&(heaptup->t_self)),
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_modified = true;
+
+ PageClearAllVisible(page);
}
/*
@@ -2160,7 +2173,6 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xl_heap_insert xlrec;
xl_heap_header xlhdr;
XLogRecPtr recptr;
- Page page = BufferGetPage(buffer);
uint8 info = XLOG_HEAP_INSERT;
int bufflags = 0;
@@ -2185,7 +2197,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.offnum = ItemPointerGetOffsetNumber(&heaptup->t_self);
xlrec.flags = 0;
- if (all_visible_cleared)
+ if (clear_all_visible)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
@@ -2230,15 +2242,28 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (vmbuffer_modified)
+ XLogRegisterBuffer(HEAP_INSERT_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, info);
PageSetLSN(page, recptr);
+
+ if (vmbuffer_modified)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
+
+ /*
+ * We locked vmbuffer if clear_all_visible was true regardless of whether
+ * or not we ended up modifying the vmbuffer.
+ */
+ if (clear_all_visible)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ if (BufferIsValid(vmbuffer))
ReleaseBuffer(vmbuffer);
/*
@@ -2419,8 +2444,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- bool all_visible_cleared = false;
+ bool clear_all_visible = false;
bool all_frozen_set = false;
+ bool vmbuffer_modified = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2462,6 +2488,17 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
all_frozen_set = true;
+ /*
+ * If clearing all-visible, take the VM buffer lock before entering
+ * the critical section where that action will be WAL-logged. Setting
+ * the VM all-frozen is done and WAL-logged separately.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ clear_all_visible = true;
+ }
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2502,13 +2539,15 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If we're only adding already frozen rows to a previously empty
* page, mark it as all-visible.
*/
- if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
+ if (clear_all_visible)
{
- all_visible_cleared = true;
+ /* It's possible the VM bits were already clear */
+ if (visibilitymap_clear_locked(relation,
+ BufferGetBlockNumber(buffer),
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_modified = true;
+
PageClearAllVisible(page);
- visibilitymap_clear(relation,
- BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
}
else if (all_frozen_set)
PageSetAllVisible(page);
@@ -2554,10 +2593,10 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
tupledata = scratchptr;
/* check that the mutually exclusive flags are not both set */
- Assert(!(all_visible_cleared && all_frozen_set));
+ Assert(!(clear_all_visible && all_frozen_set));
xlrec->flags = 0;
- if (all_visible_cleared)
+ if (clear_all_visible)
xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
if (all_frozen_set)
xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
@@ -2623,6 +2662,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogRegisterData(xlrec, tupledata - scratch.data);
XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_HEAP, buffer,
REGBUF_STANDARD | bufflags);
+ if (vmbuffer_modified)
+ XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_VM, vmbuffer, 0);
XLogRegisterBufData(HEAP_MULTI_INSERT_BLKREF_HEAP, tupledata,
totaldatalen);
@@ -2633,10 +2674,19 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
recptr = XLogInsert(RM_HEAP2_ID, info);
PageSetLSN(page, recptr);
+ if (vmbuffer_modified)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /*
+ * We locked vmbuffer if clear_all_visible was true regardless of
+ * whether or not we ended up modifying the vmbuffer.
+ */
+ if (clear_all_visible)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
/*
* If we've frozen everything on the page, update the visibilitymap.
* We're already holding pin on the vmbuffer.
@@ -2786,12 +2836,13 @@ heap_delete(Relation relation, ItemPointer tid,
BlockNumber block;
Buffer buffer;
Buffer vmbuffer = InvalidBuffer;
+ bool vmbuffer_modified = false;
TransactionId new_xmax;
uint16 new_infomask,
new_infomask2;
bool have_tuple_lock = false;
bool iscombo;
- bool all_visible_cleared = false;
+ bool clear_all_visible = false;
HeapTuple old_key_tuple = NULL; /* replica identity of the tuple */
bool old_key_copied = false;
@@ -3040,6 +3091,13 @@ l1:
xid, LockTupleExclusive, true,
&new_xmax, &new_infomask, &new_infomask2);
+ /* Lock the VM before entering the critical section */
+ if (PageIsAllVisible(page))
+ {
+ clear_all_visible = true;
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ }
+
START_CRIT_SECTION();
/*
@@ -3051,12 +3109,14 @@ l1:
*/
PageSetPrunable(page, xid);
- if (PageIsAllVisible(page))
+ if (clear_all_visible)
{
- all_visible_cleared = true;
+ /* It's possible the VM bits were already clear */
+ if (visibilitymap_clear_locked(relation, BufferGetBlockNumber(buffer),
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_modified = true;
+
PageClearAllVisible(page);
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
}
/* store transaction information of xact deleting the tuple */
@@ -3096,7 +3156,7 @@ l1:
log_heap_new_cid(relation, &tp);
xlrec.flags = 0;
- if (all_visible_cleared)
+ if (clear_all_visible)
xlrec.flags |= XLH_DELETE_ALL_VISIBLE_CLEARED;
if (changingPart)
xlrec.flags |= XLH_DELETE_IS_PARTITION_MOVE;
@@ -3137,13 +3197,27 @@ l1:
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (vmbuffer_modified)
+ XLogRegisterBuffer(HEAP_DELETE_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE);
PageSetLSN(page, recptr);
+
+ if (vmbuffer_modified)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /*
+ * Release VM lock first, since it covers many heap blocks. We locked
+ * vmbuffer if clear_all_visible was true regardless of whether or not we
+ * ended up modifying the vmbuffer.
+ */
+ if (clear_all_visible)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
if (vmbuffer != InvalidBuffer)
@@ -3261,13 +3335,16 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
HeapTuple heaptup;
HeapTuple old_key_tuple = NULL;
bool old_key_copied = false;
- Page page;
+ Page page,
+ newpage;
BlockNumber block;
MultiXactStatus mxact_status;
Buffer buffer,
newbuf,
vmbuffer = InvalidBuffer,
vmbuffer_new = InvalidBuffer;
+ bool unlock_vmbuffer = false;
+ bool unlock_vmbuffer_new = false;
bool need_toast;
Size newtupsize,
pagefree;
@@ -3276,8 +3353,10 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
bool use_hot_update = false;
bool summarized_update = false;
bool key_intact;
- bool all_visible_cleared = false;
- bool all_visible_cleared_new = false;
+ bool clear_all_visible = false;
+ bool clear_all_visible_new = false;
+ bool vmbuffer_modified = false;
+ bool vmbuffer_new_modified = false;
bool checked_lockers;
bool locker_remains;
bool id_has_external = false;
@@ -3852,6 +3931,12 @@ l2:
Assert(HEAP_XMAX_IS_LOCKED_ONLY(infomask_lock_old_tuple));
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuffer = true;
+ }
+
START_CRIT_SECTION();
/* Clear obsolete visibility flags ... */
@@ -3874,10 +3959,13 @@ l2:
* overhead would be unchanged, that doesn't seem necessarily
* worthwhile.
*/
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ /* It's possible all-frozen was already clear */
+ if (visibilitymap_clear_locked(relation, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ }
MarkBufferDirty(buffer);
@@ -3896,12 +3984,24 @@ l2:
xlrec.flags =
cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (unlock_vmbuffer)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ unlock_vmbuffer = false;
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
/*
@@ -3987,6 +4087,8 @@ l2:
heaptup = newtup;
}
+ newpage = BufferGetPage(newbuf);
+
/*
* We're about to do the actual update -- check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -4050,6 +4152,69 @@ l2:
id_has_external,
&old_key_copied);
+ clear_all_visible = PageIsAllVisible(page);
+ clear_all_visible_new = newbuf != buffer && PageIsAllVisible(newpage);
+
+ /*
+ * Clear PD_ALL_VISIBLE flags and reset visibility map bits for any heap
+ * pages that were all-visible. If there are two heap pages, we may need
+ * to clear VM bits for both.
+ */
+ if (clear_all_visible && clear_all_visible_new &&
+ vmbuffer_new == vmbuffer)
+ {
+ /*
+ * This is the more complicated case: both the new and old heap pages
+ * are all-visible and both their VM bits are on the same page of the
+ * VM, so we register a single VM buffer as HEAP_UPDATE_BLKREF_VM_NEW
+ * in the WAL record. We must be careful to only lock and register one
+ * buffer, even though we modify it twice -- once for each heap
+ * block's VM bits.
+ */
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuffer_new = true;
+
+ /* We will not lock or attempt to modify old VM buffer */
+ }
+ else
+ {
+ /*
+ * In all the remaining cases, we will clear at most one heap block's
+ * VM bits per VM page.
+ */
+ Buffer vmbuffers[2] = {
+ clear_all_visible ? vmbuffer : InvalidBuffer,
+ clear_all_visible_new ? vmbuffer_new : InvalidBuffer
+ };
+
+ /*
+ * When both pages need different VM pages cleared, acquire the VM
+ * buffer locks in VM block order to avoid deadlocks between backends
+ * updating tuples in opposite directions across VM pages.
+ */
+ if (clear_all_visible && clear_all_visible_new &&
+ BufferGetBlockNumber(vmbuffers[0]) > BufferGetBlockNumber(vmbuffers[1]))
+ {
+ Buffer swap = vmbuffers[0];
+
+ vmbuffers[0] = vmbuffers[1];
+ vmbuffers[1] = swap;
+ }
+
+ Assert((!BufferIsValid(vmbuffers[0]) && !BufferIsValid(vmbuffers[1])) ||
+ vmbuffers[0] != vmbuffers[1]);
+
+ if (BufferIsValid(vmbuffers[0]))
+ LockBuffer(vmbuffers[0], BUFFER_LOCK_EXCLUSIVE);
+ if (BufferIsValid(vmbuffers[1]))
+ LockBuffer(vmbuffers[1], BUFFER_LOCK_EXCLUSIVE);
+
+ if (clear_all_visible)
+ unlock_vmbuffer = true;
+ if (clear_all_visible_new)
+ unlock_vmbuffer_new = true;
+ }
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -4086,7 +4251,6 @@ l2:
RelationPutHeapTuple(relation, newbuf, heaptup, false); /* insert new tuple */
-
/* Clear obsolete visibility flags, possibly set by ourselves above... */
oldtup.t_data->t_infomask &= ~(HEAP_XMAX_BITS | HEAP_MOVED);
oldtup.t_data->t_infomask2 &= ~HEAP_KEYS_UPDATED;
@@ -4101,19 +4265,38 @@ l2:
oldtup.t_data->t_ctid = heaptup->t_self;
/* clear PD_ALL_VISIBLE flags, reset all visibilitymap bits */
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ if (clear_all_visible)
{
- all_visible_cleared = true;
- PageClearAllVisible(BufferGetPage(buffer));
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ /* It's possible the VM bits were already clear */
+ if (visibilitymap_clear_locked(relation, block,
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ {
+ /*
+ * If both heap blocks' VM bits are on the same VM buffer,
+ * vmbuffer_new and vmbuffer will be equal. In that case only
+ * vmbuffer_new is registered in the WAL record (see
+ * log_heap_update), so attribute any change to the old heap
+ * block's VM bits to vmbuffer_new as well.
+ */
+ if (vmbuffer == vmbuffer_new)
+ vmbuffer_new_modified = true;
+ else
+ vmbuffer_modified = true;
+ }
+
+ PageClearAllVisible(page);
}
- if (newbuf != buffer && PageIsAllVisible(BufferGetPage(newbuf)))
+ if (clear_all_visible_new)
{
- all_visible_cleared_new = true;
- PageClearAllVisible(BufferGetPage(newbuf));
- visibilitymap_clear(relation, BufferGetBlockNumber(newbuf),
- vmbuffer_new, VISIBILITYMAP_VALID_BITS);
+ /*
+ * If both heap blocks' VM bits are on the same VM buffer, this will
+ * clear the new heap block's VM bits from the shared vmbuffer.
+ */
+ if (visibilitymap_clear_locked(relation, BufferGetBlockNumber(newbuf),
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_new_modified = true;
+
+ PageClearAllVisible(newpage);
}
if (newbuf != buffer)
@@ -4136,19 +4319,30 @@ l2:
}
recptr = log_heap_update(relation, buffer,
- newbuf, &oldtup, heaptup,
+ vmbuffer_modified ? vmbuffer : InvalidBuffer,
+ newbuf,
+ vmbuffer_new_modified ? vmbuffer_new : InvalidBuffer,
+ &oldtup, heaptup,
old_key_tuple,
- all_visible_cleared,
- all_visible_cleared_new);
+ clear_all_visible,
+ clear_all_visible_new);
if (newbuf != buffer)
- {
- PageSetLSN(BufferGetPage(newbuf), recptr);
- }
+ PageSetLSN(newpage, recptr);
PageSetLSN(BufferGetPage(buffer), recptr);
+
+ if (vmbuffer_modified)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
+ if (vmbuffer_new_modified)
+ PageSetLSN(BufferGetPage(vmbuffer_new), recptr);
}
END_CRIT_SECTION();
+ if (unlock_vmbuffer)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ if (unlock_vmbuffer_new)
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_UNLOCK);
+
if (newbuf != buffer)
LockBuffer(newbuf, BUFFER_LOCK_UNLOCK);
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
@@ -4586,6 +4780,7 @@ heap_lock_tuple(Relation relation, HeapTuple tuple,
ItemId lp;
Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool unlock_vmbuffer = false;
BlockNumber block;
TransactionId xid,
xmax;
@@ -5166,6 +5361,13 @@ failed:
GetCurrentTransactionId(), mode, false,
&xid, &new_infomask, &new_infomask2);
+ /* Lock VM buffer before entering critical section */
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuffer = true;
+ }
+
START_CRIT_SECTION();
/*
@@ -5197,11 +5399,13 @@ failed:
tuple->t_data->t_ctid = *tid;
/* Clear only the all-frozen bit on visibility map if needed */
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
-
+ if (PageIsAllVisible(page))
+ {
+ /* It's possible all-frozen was already clear */
+ if (visibilitymap_clear_locked(relation, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ }
MarkBufferDirty(*buffer);
@@ -5232,19 +5436,31 @@ failed:
xlrec.flags = cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
/* we don't decode row locks atm, so no need to log the origin */
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (unlock_vmbuffer)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ unlock_vmbuffer = false;
+
result = TM_Ok;
out_locked:
LockBuffer(*buffer, BUFFER_LOCK_UNLOCK);
+ Assert(!unlock_vmbuffer);
out_unlocked:
if (BufferIsValid(vmbuffer))
@@ -5707,6 +5923,7 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
ItemPointerData tupid;
HeapTupleData mytup;
Buffer buf;
+ Page page;
uint16 new_infomask,
new_infomask2,
old_infomask,
@@ -5716,6 +5933,7 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
bool cleared_all_frozen = false;
bool pinned_desired_page;
Buffer vmbuffer = InvalidBuffer;
+ bool unlock_vmbuffer = false;
BlockNumber block;
ItemPointerCopy(tid, &tupid);
@@ -5724,6 +5942,7 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
{
new_infomask = 0;
new_xmax = InvalidTransactionId;
+ cleared_all_frozen = false;
block = ItemPointerGetBlockNumber(&tupid);
ItemPointerCopy(&tupid, &(mytup.t_self));
@@ -5743,13 +5962,15 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
l4:
CHECK_FOR_INTERRUPTS();
+ page = BufferGetPage(buf);
+
/*
* Before locking the buffer, pin the visibility map page if it
* appears to be necessary. Since we haven't got the lock yet,
* someone else might be in the middle of changing this, so we'll need
* to recheck after we have the lock.
*/
- if (PageIsAllVisible(BufferGetPage(buf)))
+ if (PageIsAllVisible(page))
{
visibilitymap_pin(rel, block, &vmbuffer);
pinned_desired_page = true;
@@ -5770,7 +5991,7 @@ l4:
* this page. If this page isn't all-visible, we won't use the vm
* page, but we hold onto such a pin till the end of the function.
*/
- if (!pinned_desired_page && PageIsAllVisible(BufferGetPage(buf)))
+ if (!pinned_desired_page && PageIsAllVisible(page))
{
LockBuffer(buf, BUFFER_LOCK_UNLOCK);
visibilitymap_pin(rel, block, &vmbuffer);
@@ -5951,10 +6172,11 @@ l4:
xid, mode, false,
&new_xmax, &new_infomask, &new_infomask2);
- if (PageIsAllVisible(BufferGetPage(buf)) &&
- visibilitymap_clear(rel, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuffer = true;
+ }
START_CRIT_SECTION();
@@ -5967,12 +6189,19 @@ l4:
MarkBufferDirty(buf);
+ if (PageIsAllVisible(page))
+ {
+ /* It's possible all-frozen was already clear */
+ if (visibilitymap_clear_locked(rel, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ }
+
/* XLOG stuff */
if (RelationNeedsWAL(rel))
{
xl_heap_lock_updated xlrec;
XLogRecPtr recptr;
- Page page = BufferGetPage(buf);
XLogBeginInsert();
XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buf, REGBUF_STANDARD);
@@ -5985,13 +6214,26 @@ l4:
XLogRegisterData(&xlrec, SizeOfHeapLockUpdated);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_LOCK_UPDATED);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (unlock_vmbuffer)
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ unlock_vmbuffer = false;
+ }
+
next:
/* if we find the end of update chain, we're done. */
if (mytup.t_data->t_infomask & HEAP_XMAX_INVALID ||
@@ -8848,8 +9090,9 @@ log_heap_visible(Relation rel, Buffer heap_buffer, Buffer vm_buffer,
* have modified the buffer(s) and marked them dirty.
*/
static XLogRecPtr
-log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup, HeapTuple newtup,
+log_heap_update(Relation reln, Buffer oldbuf, Buffer vmbuffer_old,
+ Buffer newbuf, Buffer vmbuffer_new,
+ HeapTuple oldtup, HeapTuple newtup,
HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared)
{
@@ -9058,6 +9301,20 @@ log_heap_update(Relation reln, Buffer oldbuf,
old_key_tuple->t_len - SizeofHeapTupleHeader);
}
+ /*
+ * Register VM buffers. If the old and new heap pages' VM bits are on the
+ * same VM page, the caller passes only vmbuffer_new (mirroring the heap
+ * page convention where block 0 = new is always registered).
+ */
+ Assert((BufferIsInvalid(vmbuffer_old) && BufferIsInvalid(vmbuffer_new)) ||
+ (vmbuffer_old != vmbuffer_new));
+
+ if (BufferIsValid(vmbuffer_new))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_NEW, vmbuffer_new, 0);
+
+ if (BufferIsValid(vmbuffer_old))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_OLD, vmbuffer_old, 0);
+
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index 87b5c44fd46..41bd03e80a0 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -22,6 +22,70 @@
#include "storage/freespace.h"
#include "storage/standby.h"
+/*
+ * Clear visibility map bits for a single heap block during heap redo.
+ *
+ * Used by records that modify one heap block and, at most, its corresponding
+ * VM block (insert, delete, multi_insert, lock). Records that can touch
+ * multiple heap or VM blocks (e.g. updates) replay the VM changes inline
+ * instead.
+ *
+ * 'record' is the WAL record being replayed
+ * 'target_locator' identifies the relation whose VM is being updated
+ * 'heap_blkno' is the heap block whose VM bits should be cleared
+ * 'wal_vm_block_id' is the WAL block reference id of the VM page
+ * 'flags' specifies which visibility map bits to clear
+ */
+static void
+heap_xlog_vm_clear(XLogReaderState *record,
+ RelFileLocator target_locator,
+ BlockNumber heap_blkno,
+ uint8 wal_vm_block_id, uint8 flags)
+{
+ XLogRecPtr lsn = record->EndRecPtr;
+ Relation reln = CreateFakeRelcacheEntry(target_locator);
+ Buffer vmbuffer = InvalidBuffer;
+
+ /*
+ * If the vmbuffer was registered, use the recovery-specific routines to
+ * read it. These will either apply an FPI or indicate that we should
+ * clear the requested bits ourselves.
+ *
+ * Originally, clearing the VM did not register the VM buffers, so since
+ * registering the VM buffer was a bug fix, we keep a fallback path to
+ * support replay of WAL generated from before the fix.
+ */
+ if (XLogRecHasBlockRef(record, wal_vm_block_id))
+ {
+ if (XLogReadBufferForRedo(record, wal_vm_block_id,
+ &vmbuffer) == BLK_NEEDS_REDO)
+ {
+ if (visibilitymap_clear_locked(reln,
+ heap_blkno, vmbuffer,
+ flags))
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else
+ {
+ /*
+ * This is the backwards compatibility path to clear VM bits for
+ * records predating VM buffer registration. It is also invoked if the
+ * heap page's PD_ALL_VISIBLE was cleared but the VM bits were already
+ * clear. The WAL record flags do not distinguish between these two
+ * situations. Though this is wasted effort, the behavior is
+ * historical and the situation should be rare.
+ */
+ visibilitymap_pin(reln, heap_blkno, &vmbuffer);
+ visibilitymap_clear(reln, heap_blkno, vmbuffer, flags);
+ ReleaseBuffer(vmbuffer);
+ }
+
+ FreeFakeRelcacheEntry(reln);
+}
+
/*
* Replay XLOG_HEAP2_PRUNE_* records.
@@ -360,15 +424,9 @@ heap_xlog_delete(XLogReaderState *record)
* already up-to-date.
*/
if (xlrec->flags & XLH_DELETE_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(target_locator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
+ heap_xlog_vm_clear(record, target_locator,
+ blkno, HEAP_DELETE_BLKREF_VM,
+ VISIBILITYMAP_VALID_BITS);
if (XLogReadBufferForRedo(record, HEAP_DELETE_BLKREF_HEAP,
&buffer) == BLK_NEEDS_REDO)
@@ -449,15 +507,9 @@ heap_xlog_insert(XLogReaderState *record)
* already up-to-date.
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(target_locator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
+ heap_xlog_vm_clear(record, target_locator,
+ blkno, HEAP_INSERT_BLKREF_VM,
+ VISIBILITYMAP_VALID_BITS);
/*
* If we inserted the first and only tuple on the page, re-initialize the
@@ -573,17 +625,15 @@ heap_xlog_multi_insert(XLogReaderState *record)
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
+ *
+ * Clear the VM (if needed) before clearing the heap page-level visibility
+ * flag (PD_ALL_VISIBLE) to prevent the heap page from being marked
+ * all-visible in the VM while its PD_ALL_VISIBLE is clear.
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
+ heap_xlog_vm_clear(record, rlocator,
+ blkno, HEAP_MULTI_INSERT_BLKREF_VM,
+ VISIBILITYMAP_VALID_BITS);
if (isinit)
{
@@ -698,6 +748,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Buffer obuffer,
nbuffer;
Page page;
+ bool new_cleared,
+ old_cleared;
OffsetNumber offnum;
ItemId lp = NULL;
HeapTupleData oldtup;
@@ -737,14 +789,92 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
*/
- if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED)
+ new_cleared = (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0;
+ old_cleared = (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) != 0;
+ if (new_cleared || old_cleared)
{
Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
+ bool has_vm_old = XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_OLD);
+ bool has_vm_new = XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_NEW);
+
+ if (has_vm_new)
+ {
+ Buffer vmbuffer_new = InvalidBuffer;
+ bool vmbuffer_new_modified = false;
+
+ Assert(new_cleared);
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW, &vmbuffer_new) ==
+ BLK_NEEDS_REDO)
+ {
+ /*
+ * If both the old and new heap pages were all-visible and
+ * their VM bits are on the same VM page, that single VM page
+ * is registered as HEAP_UPDATE_BLKREF_VM_NEW. Clear both heap
+ * blocks' VM bits from the single provided VM buffer.
+ *
+ * We must verify that oldblk's VM bits really are on this VM
+ * page, rather than relying on the absence of a separate
+ * VM_OLD block reference: VM_OLD is also omitted when oldblk
+ * is on a different VM page but its bit was already clear.
+ */
+ if (old_cleared && visibilitymap_pin_ok(oldblk, vmbuffer_new))
+ {
+ if (visibilitymap_clear_locked(reln, oldblk, vmbuffer_new,
+ VISIBILITYMAP_VALID_BITS))
+ vmbuffer_new_modified = true;
+ }
+ if (visibilitymap_clear_locked(reln, newblk, vmbuffer_new,
+ VISIBILITYMAP_VALID_BITS))
+ vmbuffer_new_modified = true;
+
+ if (vmbuffer_new_modified)
+ PageSetLSN(BufferGetPage(vmbuffer_new), lsn);
+ }
+ if (BufferIsValid(vmbuffer_new))
+ UnlockReleaseBuffer(vmbuffer_new);
+ }
+ if (has_vm_old)
+ {
+ Buffer vmbuffer_old = InvalidBuffer;
+
+ Assert(old_cleared);
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_OLD, &vmbuffer_old) ==
+ BLK_NEEDS_REDO)
+ {
+ if (visibilitymap_clear_locked(reln, oldblk, vmbuffer_old,
+ VISIBILITYMAP_VALID_BITS))
+ PageSetLSN(BufferGetPage(vmbuffer_old), lsn);
+ }
+ if (BufferIsValid(vmbuffer_old))
+ UnlockReleaseBuffer(vmbuffer_old);
+ }
+ if (!has_vm_old && !has_vm_new)
+ {
+ /*
+ * Backwards compatibility path. Previously, the VM buffers were
+ * not registered in the WAL record. We need this path to replay
+ * WAL generated by a not-yet-patched primary during upgrade.
+ */
+ if (old_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ visibilitymap_pin(reln, oldblk, &vmbuffer);
+ visibilitymap_clear(reln, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
+ if (new_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ visibilitymap_pin(reln, newblk, &vmbuffer);
+ visibilitymap_clear(reln, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
+ }
- visibilitymap_pin(reln, oldblk, &vmbuffer);
- visibilitymap_clear(reln, oldblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -793,7 +923,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
/* Mark the page as a candidate for pruning */
PageSetPrunable(page, XLogRecGetXid(record));
- if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED)
+ if (old_cleared)
PageClearAllVisible(page);
PageSetLSN(page, lsn);
@@ -819,21 +949,6 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
newaction = XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_HEAP_NEW,
&nbuffer);
- /*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
- */
- if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, newblk, &vmbuffer);
- visibilitymap_clear(reln, newblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
-
/* Deal with new tuple */
if (newaction == BLK_NEEDS_REDO)
{
@@ -930,7 +1045,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
if (offnum == InvalidOffsetNumber)
elog(PANIC, "failed to add tuple");
- if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED)
+ if (new_cleared)
PageClearAllVisible(page);
freespace = PageGetHeapFreeSpace(page); /* needed to update FSM below */
@@ -1022,20 +1137,15 @@ heap_xlog_lock(XLogReaderState *record)
*/
if (xlrec->flags & XLH_LOCK_ALL_FROZEN_CLEARED)
{
- RelFileLocator rlocator;
- Buffer vmbuffer = InvalidBuffer;
BlockNumber block;
- Relation reln;
+ RelFileLocator rlocator;
XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
&block);
- reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ heap_xlog_vm_clear(record, rlocator,
+ block, HEAP_LOCK_BLKREF_VM,
+ VISIBILITYMAP_ALL_FROZEN);
}
if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
@@ -1100,20 +1210,15 @@ heap_xlog_lock_updated(XLogReaderState *record)
*/
if (xlrec->flags & XLH_LOCK_ALL_FROZEN_CLEARED)
{
- RelFileLocator rlocator;
- Buffer vmbuffer = InvalidBuffer;
BlockNumber block;
- Relation reln;
+ RelFileLocator rlocator;
XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
&block);
- reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ heap_xlog_vm_clear(record, rlocator,
+ block, HEAP_LOCK_BLKREF_VM,
+ VISIBILITYMAP_ALL_FROZEN);
}
if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 1874a3fda37..d44620eca7f 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -135,9 +135,38 @@ static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks);
* You must pass a buffer containing the correct map page to this function.
* Call visibilitymap_pin first to pin the right one. This function doesn't do
* any I/O. Returns true if any bits have been cleared and false otherwise.
+ *
+ * Most callers should use visibilitymap_clear_locked rather than this
+ * function. It is usually necessary to register the VM buffer in the WAL
+ * record, and this necessitates holding the lock for longer than it is held
+ * here. However, we retain this function for behavioral compatibility on the
+ * back branches (out-of-tree callers may expect it to take the lock). And,
+ * since we have it, we use it for in-tree callers that don't have to manage
+ * the VM buffer lock themselves.
*/
bool
visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
+{
+ bool cleared = false;
+
+ LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+
+ cleared = visibilitymap_clear_locked(rel, heapBlk, vmbuf, flags);
+
+ LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
+
+ return cleared;
+}
+
+/*
+ * Clear specified bits, caller holds VM lock
+ *
+ * Like visibilitymap_clear(), except the caller must already hold the VM
+ * buffer exclusive lock and is responsible for unlocking it.
+ * Returns true if any bits were actually cleared, false otherwise.
+ */
+bool
+visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
{
BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
@@ -157,7 +186,8 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
elog(ERROR, "wrong buffer passed to visibilitymap_clear");
- LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+ Assert(BufferIsExclusiveLocked(vmbuf));
+
map = PageGetContents(BufferGetPage(vmbuf));
if (map[mapByte] & mask)
@@ -168,8 +198,6 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
cleared = true;
}
- LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
-
return cleared;
}
diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl
index 0f98c7df82e..fcb701e1b98 100644
--- a/src/bin/pg_walsummary/t/002_blocks.pl
+++ b/src/bin/pg_walsummary/t/002_blocks.pl
@@ -93,13 +93,14 @@ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
split(m@/@, $end_lsn);
ok(-f $filename, "WAL summary file exists");
-# Run pg_walsummary on it. We expect exactly two blocks to be modified,
-# block 0 and one other.
+# Run pg_walsummary on it. We expect exactly three blocks to be modified,
+# block 0 (old tuple), another block (new tuple), and the block for the VM.
my ($stdout, $stderr) = run_command([ 'pg_walsummary', '-i', $filename ]);
note($stdout);
@lines = split(/\n/, $stdout);
like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified");
+like($stdout, qr/FORK vm: block 0$/m, "stdout shows VM block 0 modified");
is($stderr, '', 'stderr is empty');
-is(0 + @lines, 2, "UPDATE modified 2 blocks");
+is(0 + @lines, 3, "UPDATE modified 3 blocks");
done_testing();
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 92744c53899..c4c3c3f15fa 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -111,6 +111,7 @@
/* This is what we need to know about delete */
#define HEAP_DELETE_BLKREF_HEAP 0
+#define HEAP_DELETE_BLKREF_VM 1
typedef struct xl_heap_delete
{
@@ -160,6 +161,7 @@ typedef struct xl_heap_header
/* This is what we need to know about insert */
#define HEAP_INSERT_BLKREF_HEAP 0
+#define HEAP_INSERT_BLKREF_VM 1
typedef struct xl_heap_insert
{
@@ -179,6 +181,7 @@ typedef struct xl_heap_insert
* (XLOG_HEAP_INIT_PAGE).
*/
#define HEAP_MULTI_INSERT_BLKREF_HEAP 0
+#define HEAP_MULTI_INSERT_BLKREF_VM 1
/*
* In HEAP_MULTI_INSERT_BLKREF_HEAP's data portion, there is an
@@ -222,10 +225,21 @@ typedef struct xl_multi_insert_tuple
*
* HEAP_UPDATE_BLKREF_HEAP_OLD: old page, if different. (no data, just a reference
* to the block)
+ *
+ * HEAP_UPDATE_BLKREF_VM_NEW: VM page covering the new heap page. Registered
+ * when XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED is set and the new heap page's VM
+ * bit was actually cleared. Also covers the old heap page's VM bits when both
+ * heap pages map to the same VM page.
+ *
+ * HEAP_UPDATE_BLKREF_VM_OLD: VM page covering the old heap page. Only
+ * registered when XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED is set, the old heap
+ * page's VM bits are on a different VM page from the new heap page's, and the
+ * old heap page's VM bit was actually cleared.
*/
-
#define HEAP_UPDATE_BLKREF_HEAP_NEW 0
#define HEAP_UPDATE_BLKREF_HEAP_OLD 1
+#define HEAP_UPDATE_BLKREF_VM_NEW 2
+#define HEAP_UPDATE_BLKREF_VM_OLD 3
typedef struct xl_heap_update
{
@@ -406,6 +420,7 @@ typedef struct xlhp_prune_items
/* This is what we need to know about lock */
#define HEAP_LOCK_BLKREF_HEAP 0
+#define HEAP_LOCK_BLKREF_VM 1
typedef struct xl_heap_lock
{
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index ea889bf9ec7..1bc59c9ac33 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -26,6 +26,8 @@
#define VM_ALL_FROZEN(r, b, v) \
((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0)
+extern bool visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk,
+ Buffer vmbuf, uint8 flags);
extern bool visibilitymap_clear(Relation rel, BlockNumber heapBlk,
Buffer vmbuf, uint8 flags);
extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk,
--
2.47.3
v4-0003-Put-pg_combinebackup-debug-test-output-in-temp-fi.patch (text/x-patch, 4.2K) download
From a171432d65b3dd4d68dc1eba3e91c008ab98798e Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 30 Apr 2026 16:06:06 -0400
Subject: [PATCH v4 3/4] Put pg_combinebackup debug test output in temp file
Having --debug as the default option to pg_combinebackup in
init_from_backup() makes the logs extremely verbose. It is used for
debugging failures, so put the output in a temp file and only keep it
around if tests fail.
Author: Melanie Plageman <melanieplageman@gmail.com>
Disussion: https://postgr.es/m/y73coty2smkxgvelh4e32kqrt5pyn2pb7cz7x4mrz6yaxitqd5%40nzhkeazqf4bn
Backpatch-through: 17
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 83 +++++++++++++++++++++++-
1 file changed, 81 insertions(+), 2 deletions(-)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 2e31c23a6ac..3388e45b726 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -125,6 +125,8 @@ our $min_compat = 12;
# list of file reservations made by get_free_port
my @port_reservation_files;
+my @combinebackup_debug_logs;
+my %combinebackup_debug_logs_printed;
# We want to choose a server port above the range that servers typically use
# on Unix systems and below the range those systems typically use for ephemeral
@@ -898,7 +900,9 @@ pathnames that should be used for the target cluster.
To restore from an incremental backup, pass the parameter combine_with_prior
as a reference to an array of prior backup names with which this backup
-is to be combined using pg_combinebackup.
+is to be combined using pg_combinebackup. pg_combinebackup is run with
+--debug, but the output is captured in a temporary file and only copied to the
+test log if pg_combinebackup fails or if a later test fails.
Streaming replication can be enabled on this node by passing the keyword
parameter has_streaming => 1. This is disabled by default.
@@ -964,7 +968,7 @@ sub init_from_backup
}
push @combineargs, @prior_backup_path, $backup_path,
'--output' => $data_path;
- PostgreSQL::Test::Utils::system_or_bail(@combineargs);
+ _run_pg_combinebackup_with_debug_log(@combineargs);
}
elsif (defined $params{tar_program})
{
@@ -1079,6 +1083,81 @@ port = $port
return;
}
+sub _run_pg_combinebackup_with_debug_log
+{
+ my (@cmd) = @_;
+ my $debug_log = PostgreSQL::Test::Utils::tempdir('pg_combinebackup') .
+ '/debug.log';
+
+ push @combinebackup_debug_logs, $debug_log;
+
+ print("# Running: " . join(" ", @cmd) . "\n");
+ # If this pg_combinebackup invocation itself fails, print its captured
+ # output immediately. Successful invocations may still be printed later if
+ # some subsequent TAP assertion in this test script fails.
+ if (!IPC::Run::run(\@cmd, '&>' => $debug_log))
+ {
+ _print_pg_combinebackup_debug_log($debug_log);
+
+ if ($? == -1)
+ {
+ BAIL_OUT(
+ sprintf(
+ "failed to execute command \"%s\": $!", join(" ", @cmd)));
+ }
+ elsif ($? & 127)
+ {
+ BAIL_OUT(
+ sprintf(
+ "command \"%s\" died with signal %d",
+ join(" ", @cmd), $? & 127));
+ }
+ else
+ {
+ BAIL_OUT(
+ sprintf(
+ "command \"%s\" exited with value %d",
+ join(" ", @cmd), $? >> 8));
+ }
+ }
+}
+
+sub _print_pg_combinebackup_debug_log
+{
+ my ($debug_log) = @_;
+
+ return unless -e $debug_log;
+ return if $combinebackup_debug_logs_printed{$debug_log}++;
+
+ diag("pg_combinebackup debug output from $debug_log:\n" .
+ PostgreSQL::Test::Utils::slurp_file($debug_log));
+}
+
+# Keep pg_combinebackup debug logs until test exit. If pg_combinebackup itself
+# fails, its log is printed at the failure site above. If pg_combinebackup
+# succeeds but a later TAP assertion fails, print the retained logs here so the
+# regress output includes the combine step that produced the restored state.
+# Remove the logs after a clean run.
+END
+{
+ # take care not to change the script's exit value
+ my $exit_code = $?;
+
+ if ($exit_code == 0 && PostgreSQL::Test::Utils::all_tests_passing())
+ {
+ unlink @combinebackup_debug_logs;
+ }
+ else
+ {
+ foreach my $debug_log (@combinebackup_debug_logs)
+ {
+ _print_pg_combinebackup_debug_log($debug_log);
+ }
+ }
+
+ $? = $exit_code;
+}
+
=pod
=item $node->rotate_logfile()
--
2.47.3
v4-0004-Test-that-VM-clear-registers-VM-buffers.patch (text/x-patch, 10.1K) download
From 7dba64d8dd71d2c1e30bf978781a399f99b1676e Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 23 Apr 2026 11:13:23 -0400
Subject: [PATCH v4 4/4] Test that VM clear registers VM buffers
The WAL summarizer only tracks registered buffers, so unregistered VM
clears are ommitted from incremental backups, corrupting the restored
visibility map. Test those cases are now fixed.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Andrey Borodin <x4mmm@yandex-team.ru>
Discussion: https://postgr.es/m/oqcsevg35xjan2327x5kdfth6q4fgeqboxfo3v3imeyih2uiny%406sez5dzxl6nt
Backpatch-through: 17
---
src/bin/pg_combinebackup/Makefile | 2 +
src/bin/pg_combinebackup/meson.build | 1 +
.../pg_combinebackup/t/012_vm_consistency.pl | 258 ++++++++++++++++++
3 files changed, 261 insertions(+)
create mode 100644 src/bin/pg_combinebackup/t/012_vm_consistency.pl
diff --git a/src/bin/pg_combinebackup/Makefile b/src/bin/pg_combinebackup/Makefile
index 33a1f4483bf..bdba0353412 100644
--- a/src/bin/pg_combinebackup/Makefile
+++ b/src/bin/pg_combinebackup/Makefile
@@ -12,6 +12,8 @@
PGFILEDESC = "pg_combinebackup - combine incremental backups"
PGAPPICON=win32
+EXTRA_INSTALL=contrib/pg_visibility
+
subdir = src/bin/pg_combinebackup
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build
index bbc4c5735ba..757b78e2fe8 100644
--- a/src/bin/pg_combinebackup/meson.build
+++ b/src/bin/pg_combinebackup/meson.build
@@ -39,6 +39,7 @@ tests += {
't/009_no_full_file.pl',
't/010_hardlink.pl',
't/011_ib_truncation.pl',
+ 't/012_vm_consistency.pl',
],
}
}
diff --git a/src/bin/pg_combinebackup/t/012_vm_consistency.pl b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
new file mode 100644
index 00000000000..b0810f8022a
--- /dev/null
+++ b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
@@ -0,0 +1,258 @@
+# Copyright (c) 2026, PostgreSQL Global Development Group
+#
+# Test that heap operations clearing visibility map bits (INSERT, UPDATE,
+# DELETE, SELECT FOR UPDATE, COPY) correctly register visibility map buffers,
+# since incremental backups rely on the WAL summarizer, which only tracks
+# registered buffers.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir_short();
+my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
+
+# Set up primary with WAL summarization enabled.
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(allows_streaming => 1);
+$primary->append_conf('postgresql.conf', <<EOF);
+summarize_wal = on
+autovacuum = off
+EOF
+$primary->start;
+
+$primary->safe_psql('postgres', q{CREATE EXTENSION pg_visibility});
+
+my @tests = (
+ {
+ label => 'INSERT',
+ table => 'vm_insert_test',
+ setup => q{CREATE TABLE vm_insert_test (id int);
+ INSERT INTO vm_insert_test DEFAULT VALUES;},
+ modify => q{INSERT INTO vm_insert_test VALUES (1)},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'DELETE',
+ table => 'vm_delete_test',
+ setup => q{CREATE TABLE vm_delete_test (id int);
+ INSERT INTO vm_delete_test VALUES (1), (2);},
+ modify => q{DELETE FROM vm_delete_test WHERE id = 1},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'UPDATE',
+ table => 'vm_update_test',
+ # Include both same-page and cross-page updates. val is stored PLAIN
+ # so the large update stays inline.
+ setup => q{CREATE TABLE vm_update_test (id INT, val TEXT);
+ ALTER TABLE vm_update_test ALTER COLUMN val SET STORAGE PLAIN;
+ INSERT INTO vm_update_test VALUES (1, 'same page'), (2, 'cross page');
+ INSERT INTO vm_update_test SELECT i, repeat('a', 200)
+ FROM generate_series(3, 70) i;},
+ modify => q{UPDATE vm_update_test SET id = 0 WHERE id = 1;
+ UPDATE vm_update_test SET val = repeat('b', 4000) WHERE id = 2;},
+ # Confirm the small update stays on the same heap page while the large
+ # update relocates the tuple to a different heap page.
+ ctid_checks => [
+ {
+ before => 'id = 1',
+ after => 'id = 0',
+ same => 1,
+ desc => 'small update stays on the same heap page',
+ },
+ {
+ before => 'id = 2',
+ after => 'id = 2',
+ same => 0,
+ desc => 'large update moves the tuple to a different heap page',
+ },
+ ],
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'LOCK',
+ table => 'vm_lock_test',
+ setup => q{CREATE TABLE vm_lock_test (id int);
+ INSERT INTO vm_lock_test VALUES (1), (2);},
+ modify => q{SELECT * FROM vm_lock_test WHERE id = 1 FOR UPDATE},
+ visible_op => '==',
+ frozen_op => '<',
+ },
+ {
+ label => 'COPY',
+ table => 'vm_copy_test',
+ setup => q{CREATE TABLE vm_copy_test (id int);
+ INSERT INTO vm_copy_test DEFAULT VALUES;},
+ modify => q{COPY vm_copy_test FROM PROGRAM 'echo 42'},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+);
+
+sub get_vm_summary
+{
+ my ($node, $table) = @_;
+ my $result = $node->safe_psql('postgres',
+ "SELECT all_visible, all_frozen FROM pg_visibility_map_summary('$table')");
+ my @vals = split(/\|/, $result);
+ return @vals;
+}
+
+# Return the heap block number of the (single) row matching $where. The ctid
+# is "(block,offset)"; casting it through point lets us pull out the block.
+sub heap_block
+{
+ my ($node, $table, $where) = @_;
+ return $node->safe_psql('postgres',
+ "SELECT (ctid::text::point)[0]::int FROM $table WHERE $where");
+}
+
+# Confirm VACUUM (FREEZE) set VM bits before testing whether later heap
+# modifications clear those bits and are captured by incremental backup. We
+# could perhaps be more exact than > 0, but the coarseness attempts to avoid
+# tset flakes.
+sub check_vacuumed_vm
+{
+ my ($node, $test) = @_;
+ my ($all_visible, $all_frozen) = get_vm_summary($node, $test->{table});
+
+ cmp_ok($all_visible, '>', 0,
+ "$test->{label} test: pages are all-visible after vacuum");
+ cmp_ok($all_frozen, '>', 0,
+ "$test->{label} test: pages are all-frozen after vacuum");
+
+ return ($all_visible, $all_frozen);
+}
+
+# Check the VM bit counts after a heap modification against the post-vacuum
+# baseline. Most operations clear both bits; tuple locking clears all-frozen
+# without clearing all-visible.
+sub check_modified_vm
+{
+ my ($node, $test) = @_;
+ my ($post_visible, $post_frozen) = get_vm_summary($node, $test->{table});
+
+ cmp_ok($post_visible, $test->{visible_op}, $test->{pre_visible},
+ "$test->{label} test: all-visible state after modification");
+ cmp_ok($post_frozen, $test->{frozen_op}, $test->{pre_frozen},
+ "$test->{label} test: all-frozen state after modification");
+
+ return ($post_visible, $post_frozen);
+}
+
+# Verify the combined backup restored VM state exactly as it exists on the
+# primary, and ask pg_visibility to check that visible tuples are consistent.
+sub validate_restored_vm
+{
+ my ($restored, $test) = @_;
+
+ my ($primary_visible, $primary_frozen) =
+ get_vm_summary($primary, $test->{table});
+ my ($restored_visible, $restored_frozen) =
+ get_vm_summary($restored, $test->{table});
+
+ is($restored_visible, $primary_visible,
+ "$test->{label} test: restored all_visible count matches primary");
+ is($restored_frozen, $primary_frozen,
+ "$test->{label} test: restored all_frozen count matches primary");
+
+ my $corrupt_tids = $restored->safe_psql('postgres',
+ "SELECT count(*) FROM pg_check_visible('$test->{table}')");
+ is($corrupt_tids, '0',
+ "$test->{label} test: no VM corruption detected by pg_check_visible");
+}
+
+# Create and populate the tables, then vacuum freeze them to set the VM bits
+foreach my $test (@tests)
+{
+ $primary->safe_psql('postgres', $test->{setup});
+ $primary->safe_psql('postgres', "VACUUM (FREEZE) $test->{table}");
+ ($test->{pre_visible}, $test->{pre_frozen}) =
+ check_vacuumed_vm($primary, $test);
+}
+
+# Take a full backup
+my $full_name = 'full';
+my $full_path = $primary->backup_dir . "/$full_name";
+$primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $full_path,
+ '--checkpoint' => 'fast',
+ ],
+ 'full backup');
+
+# Modify the tables and check that the VM bits are as expected for that test
+# after the specified modification.
+foreach my $test (@tests)
+{
+ # Record the heap block of any rows whose same-/cross-page movement we
+ # want to verify, before the modification relocates them.
+ foreach my $check (@{ $test->{ctid_checks} // [] })
+ {
+ $check->{before_blk} =
+ heap_block($primary, $test->{table}, $check->{before});
+ }
+
+ $primary->safe_psql('postgres', $test->{modify});
+ ($test->{post_visible}, $test->{post_frozen}) =
+ check_modified_vm($primary, $test);
+
+ # Confirm each checked row did (or did not) move to a different heap page.
+ foreach my $check (@{ $test->{ctid_checks} // [] })
+ {
+ my $after_blk = heap_block($primary, $test->{table}, $check->{after});
+ if ($check->{same})
+ {
+ is($after_blk, $check->{before_blk},
+ "$test->{label} test: $check->{desc}");
+ }
+ else
+ {
+ isnt($after_blk, $check->{before_blk},
+ "$test->{label} test: $check->{desc}");
+ }
+ }
+}
+
+# Take an incremental backup. This will have the changes made in the
+# modification step.
+my $incr_name = 'incr';
+my $incr_path = $primary->backup_dir . "/$incr_name";
+$primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $incr_path,
+ '--checkpoint' => 'fast',
+ '--incremental' => $full_path . '/backup_manifest',
+ ],
+ 'incremental backup');
+
+# Start a server from a combined backup composed of the incremental and full
+# backup.
+my $restored = PostgreSQL::Test::Cluster->new('restored');
+$restored->init_from_backup($primary, $incr_name,
+ combine_with_prior => [$full_name],
+ combine_mode => $mode);
+$restored->append_conf('postgresql.conf', <<EOF);
+autovacuum = off
+EOF
+$restored->start;
+$restored->safe_psql('postgres', q{CREATE EXTENSION IF NOT EXISTS pg_visibility});
+
+# Confirm that the restored server's visibility map matches the original server
+foreach my $test (@tests)
+{
+ validate_restored_vm($restored, $test);
+}
+
+$restored->stop;
+$primary->stop;
+
+done_testing();
--
2.47.3
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-10 19:12 Melanie Plageman <melanieplageman@gmail.com>
parent: Andrey Borodin <x4mmm@yandex-team.ru>
0 siblings, 0 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-07-10 19:12 UTC (permalink / raw)
To: Andrey Borodin <x4mmm@yandex-team.ru>; +Cc: Robert Haas <robertmhaas@gmail.com>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Mon, Jun 29, 2026 at 3:47 AM Andrey Borodin <x4mmm@yandex-team.ru> wrote:
>
> I've done some mechanical checks on the 18 variant: the new 012_vm_consistency test is green
> (and goes red without the fix), and minor-version WAL compatibility holds:
> WAL replays both ways between patched and unpatched 18.4 (patched replays
> old-format records via the fallback path; unpatched harmlessly ignores the added
> VM block refs).
Thanks for the review.
> Did I get it right that we will not have this
>
> + * Backwards compatibility path. Previously, the VM buffers were
>
> on v19?
That's correct. The code in replay which clears the VM even if the VM
buffers aren't registered won't be there for 19/20.
> - The new t/012_vm_consistency.pl needs pg_visibility, but
> src/bin/pg_combinebackup/Makefile doesn't declare
> EXTRA_INSTALL=contrib/pg_visibility, so a standalone
> "make -C src/bin/pg_combinebackup check" fails with "extension pg_visibility
> is not available". (meson test and make check-world are fine, since they install
> the whole tree) (for example see how src/bin/pg_amcheck pulls amcheck/pageinspect)
Whoops! Thanks for finding this
> - A tiny wording nit: the comment on the retained visibilitymap_clear()
> says it's kept "for backwards compatibility". Maybe spell out that it's kept for
> ABI(prototype stability) on the back branches?
In the v4 I posted in a previous email, I updated the comment. The
funny thing is I actually didn't mean backwards compatibility to mean
prototype stability in the original comment you are referring to. I
meant that we use that function it in the backwards compatibility code
path, but it is true that we wouldn't want to change the behavior (not
taking locks) of an exposed function in back branches. Comment should
describe this now.
- Melanie
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-14 19:42 Melanie Plageman <melanieplageman@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 1 reply; 57+ messages in thread
From: Melanie Plageman @ 2026-07-14 19:42 UTC (permalink / raw)
To: Robert Haas <robertmhaas@gmail.com>; +Cc: Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Fri, Jul 10, 2026 at 3:06 PM Melanie Plageman
<melanieplageman@gmail.com> wrote:
>
> For 0003, I am much less confident. I've never written test
> infrastructure in Perl before. I used AI to help me write it, so I
> could really use review from someone experienced with adding TAP test
> utility functions. A few thoughts on it/questions:
I could still use review on 0003.
I hadn't checked CI before posting v4, and it caught a bug. Attached
v5 (based on REL_18_STABLE) fixes that bug. In heap_update(), when the
two heap pages' VM bits were on the same VM page and I was only
clearing the old VM bits, I was registering the VM buffer with the
wrong label. This made me wonder if I should change the logic in
heap_update() for setting vmbuffer_[new_]modified to handle the "same
VM page and both bits clear" case separately. It duplicates code but
it could be clearer. I am on the fence. I included a file
heap_update.diff that shows what that would look like.
I've also finished all branch versions of this. Each one's code
differs, but I've managed to keep the commit messages the same, which
should help release notes. Note that REL_19_STABLE bumps
XLOG_PAGE_MAGIC. This would keep us from having to maintain the
backwards compatibility code there, which I think is what we want.
Here are those branches.
- Melanie
17: https://github.com/melanieplageman/postgres/tree/17_vm-clear-logging
18: https://github.com/melanieplageman/postgres/tree/18_vm-clear-logging
19: https://github.com/melanieplageman/postgres/tree/19_vm-clear-logging
20 (master): https://github.com/melanieplageman/postgres/tree/20_vm-clear-logging
Attachments: download all (5 files, .tar.gz)
v5-0001-Introduce-macros-for-WAL-block-reference-IDs-of-s.patch (text/x-patch, 16.0K) download
From 52df426a99b428a818fd135282328496fb56dd61 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Mon, 4 May 2026 10:41:13 -0400
Subject: [PATCH v5 1/4] Introduce macros for WAL block reference IDs of some
heap record types
When registering a buffer with the WAL machinery, the caller assigns it a
block reference ID, and replay must read each block back by that same ID.
Today these IDs are bare integers assigned by convention (0, 1, 2, ...),
which is easy to follow when a record registers a single block, or when the
blocks are handled during replay in their registration order.
An upcoming bug fix registers up to two visibility map blocks in addition
to the heap block(s) when clearing the VM, and these are not handled during
replay in a straightforward 1:1, in-registration-order fashion. Relying on
bare integers for the block IDs in that case is error-prone.
Introduce macros naming the block reference IDs for the heap record types
that the upcoming commit extends to register visibility map blocks, so the
registration and replay sites refer to the same block by a meaningful name.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Robert Haas <robertmhaas@gmail.com>
Discussion: https://postgr.es/m/66mqpfyti3qhfttcsv6r2lbvqqd32rrmpn6i47ovrsnvguts46%40gou54xc>
Backpatch through: 17
---
src/backend/access/heap/heapam.c | 43 +++++++++++---------
src/backend/access/heap/heapam_xlog.c | 56 +++++++++++++++++----------
src/include/access/heapam_xlog.h | 28 ++++++++++----
3 files changed, 82 insertions(+), 45 deletions(-)
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index aff02cb003e..cf37956a2bf 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2218,10 +2218,12 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
* write the whole page to the xlog, we don't need to store
* xl_heap_header in the xlog.
*/
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
- XLogRegisterBufData(0, &xlhdr, SizeOfHeapHeader);
+ XLogRegisterBuffer(HEAP_INSERT_BLKREF_HEAP, buffer,
+ REGBUF_STANDARD | bufflags);
+ XLogRegisterBufData(HEAP_INSERT_BLKREF_HEAP, &xlhdr,
+ SizeOfHeapHeader);
/* PG73FORMAT: write bitmap [+ padding] [+ oid] + data */
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_INSERT_BLKREF_HEAP,
(char *) heaptup->t_data + SizeofHeapTupleHeader,
heaptup->t_len - SizeofHeapTupleHeader);
@@ -2619,9 +2621,11 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogBeginInsert();
XLogRegisterData(xlrec, tupledata - scratch.data);
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
+ XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_HEAP, buffer,
+ REGBUF_STANDARD | bufflags);
- XLogRegisterBufData(0, tupledata, totaldatalen);
+ XLogRegisterBufData(HEAP_MULTI_INSERT_BLKREF_HEAP, tupledata,
+ totaldatalen);
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
@@ -3112,7 +3116,7 @@ heap_delete(Relation relation, ItemPointer tid,
XLogBeginInsert();
XLogRegisterData(&xlrec, SizeOfHeapDelete);
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_DELETE_BLKREF_HEAP, buffer, REGBUF_STANDARD);
/*
* Log replica identity of the deleted tuple if there is one
@@ -3883,7 +3887,7 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
XLogRecPtr recptr;
XLogBeginInsert();
- XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buffer, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&oldtup.t_self);
xlrec.xmax = xmax_lock_old_tuple;
@@ -5219,7 +5223,7 @@ heap_lock_tuple(Relation relation, HeapTuple tuple,
XLogRecPtr recptr;
XLogBeginInsert();
- XLogRegisterBuffer(0, *buffer, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, *buffer, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&tuple->t_self);
xlrec.xmax = xid;
@@ -5971,7 +5975,7 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
Page page = BufferGetPage(buf);
XLogBeginInsert();
- XLogRegisterBuffer(0, buf, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buf, REGBUF_STANDARD);
xlrec.offnum = ItemPointerGetOffsetNumber(&mytup.t_self);
xlrec.xmax = new_xmax;
@@ -8972,9 +8976,9 @@ log_heap_update(Relation reln, Buffer oldbuf,
if (need_tuple_data)
bufflags |= REGBUF_KEEP_DATA;
- XLogRegisterBuffer(0, newbuf, bufflags);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_HEAP_NEW, newbuf, bufflags);
if (oldbuf != newbuf)
- XLogRegisterBuffer(1, oldbuf, REGBUF_STANDARD);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_HEAP_OLD, oldbuf, REGBUF_STANDARD);
XLogRegisterData(&xlrec, SizeOfHeapUpdate);
@@ -8987,15 +8991,18 @@ log_heap_update(Relation reln, Buffer oldbuf,
{
prefix_suffix[0] = prefixlen;
prefix_suffix[1] = suffixlen;
- XLogRegisterBufData(0, &prefix_suffix, sizeof(uint16) * 2);
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW, &prefix_suffix,
+ sizeof(uint16) * 2);
}
else if (prefixlen > 0)
{
- XLogRegisterBufData(0, &prefixlen, sizeof(uint16));
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW, &prefixlen,
+ sizeof(uint16));
}
else
{
- XLogRegisterBufData(0, &suffixlen, sizeof(uint16));
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW, &suffixlen,
+ sizeof(uint16));
}
}
@@ -9009,10 +9016,10 @@ log_heap_update(Relation reln, Buffer oldbuf,
*
* The 'data' doesn't include the common prefix or suffix.
*/
- XLogRegisterBufData(0, &xlhdr, SizeOfHeapHeader);
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW, &xlhdr, SizeOfHeapHeader);
if (prefixlen == 0)
{
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW,
(char *) newtup->t_data + SizeofHeapTupleHeader,
newtup->t_len - SizeofHeapTupleHeader - suffixlen);
}
@@ -9025,13 +9032,13 @@ log_heap_update(Relation reln, Buffer oldbuf,
/* bitmap [+ padding] [+ oid] */
if (newtup->t_data->t_hoff - SizeofHeapTupleHeader > 0)
{
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW,
(char *) newtup->t_data + SizeofHeapTupleHeader,
newtup->t_data->t_hoff - SizeofHeapTupleHeader);
}
/* data after common prefix */
- XLogRegisterBufData(0,
+ XLogRegisterBufData(HEAP_UPDATE_BLKREF_HEAP_NEW,
(char *) newtup->t_data + newtup->t_data->t_hoff + prefixlen,
newtup->t_len - newtup->t_data->t_hoff - prefixlen - suffixlen);
}
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index eb4bd3d6ae3..87b5c44fd46 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -350,7 +350,8 @@ heap_xlog_delete(XLogReaderState *record)
RelFileLocator target_locator;
ItemPointerData target_tid;
- XLogRecGetBlockTag(record, 0, &target_locator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_DELETE_BLKREF_HEAP, &target_locator, NULL,
+ &blkno);
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
@@ -369,7 +370,8 @@ heap_xlog_delete(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_DELETE_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = BufferGetPage(buffer);
@@ -434,7 +436,8 @@ heap_xlog_insert(XLogReaderState *record)
ItemPointerData target_tid;
XLogRedoAction action;
- XLogRecGetBlockTag(record, 0, &target_locator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_INSERT_BLKREF_HEAP, &target_locator, NULL,
+ &blkno);
ItemPointerSetBlockNumber(&target_tid, blkno);
ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum);
@@ -462,13 +465,14 @@ heap_xlog_insert(XLogReaderState *record)
*/
if (XLogRecGetInfo(record) & XLOG_HEAP_INIT_PAGE)
{
- buffer = XLogInitBufferForRedo(record, 0);
+ buffer = XLogInitBufferForRedo(record, HEAP_INSERT_BLKREF_HEAP);
page = BufferGetPage(buffer);
PageInit(page, BufferGetPageSize(buffer), 0);
action = BLK_NEEDS_REDO;
}
else
- action = XLogReadBufferForRedo(record, 0, &buffer);
+ action = XLogReadBufferForRedo(record, HEAP_INSERT_BLKREF_HEAP,
+ &buffer);
if (action == BLK_NEEDS_REDO)
{
Size datalen;
@@ -479,7 +483,7 @@ heap_xlog_insert(XLogReaderState *record)
if (PageGetMaxOffsetNumber(page) + 1 < xlrec->offnum)
elog(PANIC, "invalid max offset number");
- data = XLogRecGetBlockData(record, 0, &datalen);
+ data = XLogRecGetBlockData(record, HEAP_INSERT_BLKREF_HEAP, &datalen);
newlen = datalen - SizeOfHeapHeader;
Assert(datalen > SizeOfHeapHeader && newlen <= MaxHeapTupleSize);
@@ -559,7 +563,8 @@ heap_xlog_multi_insert(XLogReaderState *record)
*/
xlrec = (xl_heap_multi_insert *) XLogRecGetData(record);
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &blkno);
+ XLogRecGetBlockTag(record, HEAP_MULTI_INSERT_BLKREF_HEAP, &rlocator, NULL,
+ &blkno);
/* check that the mutually exclusive flags are not both set */
Assert(!((xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED) &&
@@ -582,13 +587,14 @@ heap_xlog_multi_insert(XLogReaderState *record)
if (isinit)
{
- buffer = XLogInitBufferForRedo(record, 0);
+ buffer = XLogInitBufferForRedo(record, HEAP_MULTI_INSERT_BLKREF_HEAP);
page = BufferGetPage(buffer);
PageInit(page, BufferGetPageSize(buffer), 0);
action = BLK_NEEDS_REDO;
}
else
- action = XLogReadBufferForRedo(record, 0, &buffer);
+ action = XLogReadBufferForRedo(record, HEAP_MULTI_INSERT_BLKREF_HEAP,
+ &buffer);
if (action == BLK_NEEDS_REDO)
{
char *tupdata;
@@ -596,7 +602,8 @@ heap_xlog_multi_insert(XLogReaderState *record)
Size len;
/* Tuples are stored as block data */
- tupdata = XLogRecGetBlockData(record, 0, &len);
+ tupdata = XLogRecGetBlockData(record, HEAP_MULTI_INSERT_BLKREF_HEAP,
+ &len);
endptr = tupdata + len;
page = (Page) BufferGetPage(buffer);
@@ -713,8 +720,10 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
oldtup.t_data = NULL;
oldtup.t_len = 0;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &newblk);
- if (XLogRecGetBlockTagExtended(record, 1, NULL, NULL, &oldblk, NULL))
+ XLogRecGetBlockTag(record, HEAP_UPDATE_BLKREF_HEAP_NEW, &rlocator, NULL,
+ &newblk);
+ if (XLogRecGetBlockTagExtended(record, HEAP_UPDATE_BLKREF_HEAP_OLD, NULL, NULL,
+ &oldblk, NULL))
{
/* HOT updates are never done across pages */
Assert(!hot_update);
@@ -750,7 +759,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
*/
/* Deal with old tuple version */
- oldaction = XLogReadBufferForRedo(record, (oldblk == newblk) ? 0 : 1,
+ oldaction = XLogReadBufferForRedo(record, (oldblk == newblk) ?
+ HEAP_UPDATE_BLKREF_HEAP_NEW : HEAP_UPDATE_BLKREF_HEAP_OLD,
&obuffer);
if (oldaction == BLK_NEEDS_REDO)
{
@@ -800,13 +810,14 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
}
else if (XLogRecGetInfo(record) & XLOG_HEAP_INIT_PAGE)
{
- nbuffer = XLogInitBufferForRedo(record, 0);
+ nbuffer = XLogInitBufferForRedo(record, HEAP_UPDATE_BLKREF_HEAP_NEW);
page = (Page) BufferGetPage(nbuffer);
PageInit(page, BufferGetPageSize(nbuffer), 0);
newaction = BLK_NEEDS_REDO;
}
else
- newaction = XLogReadBufferForRedo(record, 0, &nbuffer);
+ newaction = XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_HEAP_NEW,
+ &nbuffer);
/*
* The visibility map may need to be fixed even if the heap page is
@@ -831,7 +842,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Size datalen;
Size tuplen;
- recdata = XLogRecGetBlockData(record, 0, &datalen);
+ recdata = XLogRecGetBlockData(record, HEAP_UPDATE_BLKREF_HEAP_NEW,
+ &datalen);
recdata_end = recdata + datalen;
page = BufferGetPage(nbuffer);
@@ -1015,7 +1027,8 @@ heap_xlog_lock(XLogReaderState *record)
BlockNumber block;
Relation reln;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
+ XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
+ &block);
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
@@ -1025,7 +1038,8 @@ heap_xlog_lock(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = (Page) BufferGetPage(buffer);
@@ -1091,7 +1105,8 @@ heap_xlog_lock_updated(XLogReaderState *record)
BlockNumber block;
Relation reln;
- XLogRecGetBlockTag(record, 0, &rlocator, NULL, &block);
+ XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
+ &block);
reln = CreateFakeRelcacheEntry(rlocator);
visibilitymap_pin(reln, block, &vmbuffer);
@@ -1101,7 +1116,8 @@ heap_xlog_lock_updated(XLogReaderState *record)
FreeFakeRelcacheEntry(reln);
}
- if (XLogReadBufferForRedo(record, 0, &buffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
+ &buffer) == BLK_NEEDS_REDO)
{
page = BufferGetPage(buffer);
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 277df6b3cf0..92744c53899 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -110,6 +110,8 @@
(XLH_DELETE_CONTAINS_OLD_TUPLE | XLH_DELETE_CONTAINS_OLD_KEY)
/* This is what we need to know about delete */
+#define HEAP_DELETE_BLKREF_HEAP 0
+
typedef struct xl_heap_delete
{
TransactionId xmax; /* xmax of the deleted tuple */
@@ -157,12 +159,14 @@ typedef struct xl_heap_header
#define SizeOfHeapHeader (offsetof(xl_heap_header, t_hoff) + sizeof(uint8))
/* This is what we need to know about insert */
+#define HEAP_INSERT_BLKREF_HEAP 0
+
typedef struct xl_heap_insert
{
OffsetNumber offnum; /* inserted tuple's offset */
uint8 flags;
- /* xl_heap_header & TUPLE DATA in backup block 0 */
+ /* xl_heap_header & TUPLE DATA in HEAP_INSERT_BLKREF_HEAP */
} xl_heap_insert;
#define SizeOfHeapInsert (offsetof(xl_heap_insert, flags) + sizeof(uint8))
@@ -173,10 +177,13 @@ typedef struct xl_heap_insert
* The main data of the record consists of this xl_heap_multi_insert header.
* 'offsets' array is omitted if the whole page is reinitialized
* (XLOG_HEAP_INIT_PAGE).
- *
- * In block 0's data portion, there is an xl_multi_insert_tuple struct,
- * followed by the tuple data for each tuple. There is padding to align
- * each xl_multi_insert_tuple struct.
+ */
+#define HEAP_MULTI_INSERT_BLKREF_HEAP 0
+
+/*
+ * In HEAP_MULTI_INSERT_BLKREF_HEAP's data portion, there is an
+ * xl_multi_insert_tuple struct, followed by the tuple data for each tuple.
+ * There is padding to align each xl_multi_insert_tuple struct.
*/
typedef struct xl_heap_multi_insert
{
@@ -201,7 +208,7 @@ typedef struct xl_multi_insert_tuple
/*
* This is what we need to know about update|hot_update
*
- * Backup blk 0: new page
+ * HEAP_UPDATE_BLKREF_HEAP_NEW: new page
*
* If XLH_UPDATE_PREFIX_FROM_OLD or XLH_UPDATE_SUFFIX_FROM_OLD flags are set,
* the prefix and/or suffix come first, as one or two uint16s.
@@ -213,8 +220,13 @@ typedef struct xl_multi_insert_tuple
* If XLH_UPDATE_CONTAINS_NEW_TUPLE flag is given, the tuple data is
* included even if a full-page image was taken.
*
- * Backup blk 1: old page, if different. (no data, just a reference to the blk)
+ * HEAP_UPDATE_BLKREF_HEAP_OLD: old page, if different. (no data, just a reference
+ * to the block)
*/
+
+#define HEAP_UPDATE_BLKREF_HEAP_NEW 0
+#define HEAP_UPDATE_BLKREF_HEAP_OLD 1
+
typedef struct xl_heap_update
{
TransactionId old_xmax; /* xmax of the old tuple */
@@ -393,6 +405,8 @@ typedef struct xlhp_prune_items
#define XLH_LOCK_ALL_FROZEN_CLEARED 0x01
/* This is what we need to know about lock */
+#define HEAP_LOCK_BLKREF_HEAP 0
+
typedef struct xl_heap_lock
{
TransactionId xmax; /* might be a MultiXactId */
--
2.47.3
v5-0002-Fix-VM-clear-WAL-logging-by-registering-VM-blocks.patch (text/x-patch, 49.0K) download
From edb0c2262eab9ca3515214e24813b855b76e1885 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 12:53:49 -0400
Subject: [PATCH v5 2/4] Fix VM clear WAL logging by registering VM blocks
Heap WAL records that clear bits on the visibility map (like inserts and
deletes) did not register the visibility map blocks they modified.
Because the WAL summarizer only records registered blocks, an
incremental backup taken over such operations would omit the changed VM
pages. On restore, the VM would retain stale all-visible/all-frozen
bits, which can cause wrong results from index-only scans and incorrect
relfrozenxid advancement due to vacuum page skipping.
Not registering the VM buffer also meant we never emitted FPIs of VM
pages when clearing bits. A torn VM page won't raise an error because
the VM is read with ZERO_ON_ERROR; with checksums on, it would be
detected and zeroed, but with checksums off, it is accepted as-is and
can lead to data corruption.
Fix this by registering the VM buffer in the WAL record when clearing VM
bits. The VM buffer must now be locked throughout the critical section
that modifies the VM and heap pages and emits the WAL record. This can
slow down operations that clear the VM, since the VM lock is held longer
and VM FPIs may be emitted, but it is required for correctness.
Note that this fix does not repair existing incremental backups.
Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Andres Freund <andres@anarazel.de>
Reviewed-by: Robert Haas <robertmhaas@gmail.com>
Reviewed-by: Andrey Borodin <x4mmm@yandex-team.ru>
Discussion: https://postgr.es/m/flat/CAAKRu_bn%2Be7F4yPFBgFbnP%2BsyJRKyNK092bjD2LKvZW7O4Svag>
Backpatch-through: 17
---
contrib/pg_surgery/heap_surgery.c | 40 ++-
src/backend/access/heap/heapam.c | 396 ++++++++++++++++++++----
src/backend/access/heap/heapam_xlog.c | 252 ++++++++++-----
src/backend/access/heap/visibilitymap.c | 34 +-
src/bin/pg_walsummary/t/002_blocks.pl | 7 +-
src/include/access/heapam_xlog.h | 17 +-
src/include/access/visibilitymap.h | 2 +
7 files changed, 593 insertions(+), 155 deletions(-)
diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c
index 602aca66c60..6a38ac577c6 100644
--- a/contrib/pg_surgery/heap_surgery.c
+++ b/contrib/pg_surgery/heap_surgery.c
@@ -17,6 +17,7 @@
#include "access/visibilitymap.h"
#include "access/xloginsert.h"
#include "catalog/pg_am_d.h"
+#include "catalog/pg_control.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
#include "utils/acl.h"
@@ -146,6 +147,7 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
{
Buffer buf;
Buffer vmbuf = InvalidBuffer;
+ bool unlock_vmbuf = false;
Page page;
BlockNumber blkno;
OffsetNumber curoff;
@@ -233,11 +235,15 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
}
/*
- * Before entering the critical section, pin the visibility map page
- * if it appears to be necessary.
+ * Before entering the critical section, pin and lock the visibility
+ * map page if it appears to be necessary.
*/
if (heap_force_opt == HEAP_FORCE_KILL && PageIsAllVisible(page))
+ {
visibilitymap_pin(rel, blkno, &vmbuf);
+ LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuf = true;
+ }
/* No ereport(ERROR) from here until all the changes are logged. */
START_CRIT_SECTION();
@@ -266,10 +272,11 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
*/
if (PageIsAllVisible(page))
{
+ if (visibilitymap_clear_locked(rel, blkno, vmbuf,
+ VISIBILITYMAP_VALID_BITS))
+ did_modify_vm = true;
+
PageClearAllVisible(page);
- visibilitymap_clear(rel, blkno, vmbuf,
- VISIBILITYMAP_VALID_BITS);
- did_modify_vm = true;
}
}
else
@@ -320,18 +327,29 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
/* XLOG stuff */
if (RelationNeedsWAL(rel))
- log_newpage_buffer(buf, true);
+ {
+ XLogRecPtr recptr;
+
+ XLogBeginInsert();
+ XLogRegisterBuffer(0, buf, REGBUF_STANDARD | REGBUF_FORCE_IMAGE);
+ /* Include the VM page if it was modified. */
+ if (did_modify_vm)
+ XLogRegisterBuffer(1, vmbuf, REGBUF_FORCE_IMAGE);
+ recptr = XLogInsert(RM_XLOG_ID, XLOG_FPI);
+ if (did_modify_vm)
+ PageSetLSN(BufferGetPage(vmbuf), recptr);
+ PageSetLSN(BufferGetPage(buf), recptr);
+ }
}
- /* WAL log the VM page if it was modified. */
- if (did_modify_vm && RelationNeedsWAL(rel))
- log_newpage_buffer(vmbuf, false);
-
END_CRIT_SECTION();
UnlockReleaseBuffer(buf);
- if (vmbuf != InvalidBuffer)
+ if (unlock_vmbuf)
+ LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
+
+ if (BufferIsValid(vmbuf))
ReleaseBuffer(vmbuf);
/* Update the current_start_ptr before moving to the next page. */
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index cf37956a2bf..fd8ca72728f 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -58,7 +58,8 @@
static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup,
TransactionId xid, CommandId cid, int options);
static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup,
+ Buffer vmbuffer_old, Buffer newbuf,
+ Buffer vmbuffer_new, HeapTuple oldtup,
HeapTuple newtup, HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared);
#ifdef USE_ASSERT_CHECKING
@@ -2083,8 +2084,10 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
TransactionId xid = GetCurrentTransactionId();
HeapTuple heaptup;
Buffer buffer;
+ Page page;
Buffer vmbuffer = InvalidBuffer;
- bool all_visible_cleared = false;
+ bool clear_all_visible = false;
+ bool vmbuffer_modified = false;
/* Cheap, simplistic check that the tuple matches the rel's rowtype. */
Assert(HeapTupleHeaderGetNatts(tup->t_data) <=
@@ -2108,6 +2111,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
InvalidBuffer, options, bistate,
&vmbuffer, NULL,
0);
+ page = BufferGetPage(buffer);
/*
* We're about to do the actual insert -- but check for conflict first, to
@@ -2126,19 +2130,28 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
*/
CheckForSerializableConflictIn(relation, NULL, InvalidBlockNumber);
+ /* Lock the vmbuffer before the critical section */
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ clear_all_visible = true;
+ }
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
RelationPutHeapTuple(relation, buffer, heaptup,
(options & HEAP_INSERT_SPECULATIVE) != 0);
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ if (clear_all_visible)
{
- all_visible_cleared = true;
- PageClearAllVisible(BufferGetPage(buffer));
- visibilitymap_clear(relation,
- ItemPointerGetBlockNumber(&(heaptup->t_self)),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ /* It's possible the VM bits were already clear */
+ if (visibilitymap_clear_locked(relation,
+ ItemPointerGetBlockNumber(&(heaptup->t_self)),
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_modified = true;
+
+ PageClearAllVisible(page);
}
/*
@@ -2160,7 +2173,6 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xl_heap_insert xlrec;
xl_heap_header xlhdr;
XLogRecPtr recptr;
- Page page = BufferGetPage(buffer);
uint8 info = XLOG_HEAP_INSERT;
int bufflags = 0;
@@ -2185,7 +2197,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
xlrec.offnum = ItemPointerGetOffsetNumber(&heaptup->t_self);
xlrec.flags = 0;
- if (all_visible_cleared)
+ if (clear_all_visible)
xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
if (options & HEAP_INSERT_SPECULATIVE)
xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
@@ -2230,15 +2242,28 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (vmbuffer_modified)
+ XLogRegisterBuffer(HEAP_INSERT_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, info);
PageSetLSN(page, recptr);
+
+ if (vmbuffer_modified)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
UnlockReleaseBuffer(buffer);
- if (vmbuffer != InvalidBuffer)
+
+ /*
+ * We locked vmbuffer if clear_all_visible was true regardless of whether
+ * or not we ended up modifying the vmbuffer.
+ */
+ if (clear_all_visible)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ if (BufferIsValid(vmbuffer))
ReleaseBuffer(vmbuffer);
/*
@@ -2419,8 +2444,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
while (ndone < ntuples)
{
Buffer buffer;
- bool all_visible_cleared = false;
+ bool clear_all_visible = false;
bool all_frozen_set = false;
+ bool vmbuffer_modified = false;
int nthispage;
CHECK_FOR_INTERRUPTS();
@@ -2462,6 +2488,17 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
all_frozen_set = true;
+ /*
+ * If clearing all-visible, take the VM buffer lock before entering
+ * the critical section where that action will be WAL-logged. Setting
+ * the VM all-frozen is done and WAL-logged separately.
+ */
+ if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ clear_all_visible = true;
+ }
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -2502,13 +2539,16 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
* If we're only adding already frozen rows to a previously empty
* page, mark it as all-visible.
*/
- if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
+ if (clear_all_visible)
{
- all_visible_cleared = true;
+ Assert(!(options & HEAP_INSERT_FROZEN));
+ /* It's possible the VM bits were already clear */
+ if (visibilitymap_clear_locked(relation,
+ BufferGetBlockNumber(buffer),
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_modified = true;
+
PageClearAllVisible(page);
- visibilitymap_clear(relation,
- BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
}
else if (all_frozen_set)
PageSetAllVisible(page);
@@ -2554,10 +2594,10 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
tupledata = scratchptr;
/* check that the mutually exclusive flags are not both set */
- Assert(!(all_visible_cleared && all_frozen_set));
+ Assert(!(clear_all_visible && all_frozen_set));
xlrec->flags = 0;
- if (all_visible_cleared)
+ if (clear_all_visible)
xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
if (all_frozen_set)
xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
@@ -2623,6 +2663,8 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogRegisterData(xlrec, tupledata - scratch.data);
XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_HEAP, buffer,
REGBUF_STANDARD | bufflags);
+ if (vmbuffer_modified)
+ XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_VM, vmbuffer, 0);
XLogRegisterBufData(HEAP_MULTI_INSERT_BLKREF_HEAP, tupledata,
totaldatalen);
@@ -2633,10 +2675,19 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
recptr = XLogInsert(RM_HEAP2_ID, info);
PageSetLSN(page, recptr);
+ if (vmbuffer_modified)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /*
+ * We locked vmbuffer if clear_all_visible was true regardless of
+ * whether or not we ended up modifying the vmbuffer.
+ */
+ if (clear_all_visible)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
/*
* If we've frozen everything on the page, update the visibilitymap.
* We're already holding pin on the vmbuffer.
@@ -2786,12 +2837,13 @@ heap_delete(Relation relation, ItemPointer tid,
BlockNumber block;
Buffer buffer;
Buffer vmbuffer = InvalidBuffer;
+ bool vmbuffer_modified = false;
TransactionId new_xmax;
uint16 new_infomask,
new_infomask2;
bool have_tuple_lock = false;
bool iscombo;
- bool all_visible_cleared = false;
+ bool clear_all_visible = false;
HeapTuple old_key_tuple = NULL; /* replica identity of the tuple */
bool old_key_copied = false;
@@ -3040,6 +3092,13 @@ heap_delete(Relation relation, ItemPointer tid,
xid, LockTupleExclusive, true,
&new_xmax, &new_infomask, &new_infomask2);
+ /* Lock the VM before entering the critical section */
+ if (PageIsAllVisible(page))
+ {
+ clear_all_visible = true;
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ }
+
START_CRIT_SECTION();
/*
@@ -3051,12 +3110,14 @@ heap_delete(Relation relation, ItemPointer tid,
*/
PageSetPrunable(page, xid);
- if (PageIsAllVisible(page))
+ if (clear_all_visible)
{
- all_visible_cleared = true;
+ /* It's possible the VM bits were already clear */
+ if (visibilitymap_clear_locked(relation, BufferGetBlockNumber(buffer),
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_modified = true;
+
PageClearAllVisible(page);
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
}
/* store transaction information of xact deleting the tuple */
@@ -3096,7 +3157,7 @@ heap_delete(Relation relation, ItemPointer tid,
log_heap_new_cid(relation, &tp);
xlrec.flags = 0;
- if (all_visible_cleared)
+ if (clear_all_visible)
xlrec.flags |= XLH_DELETE_ALL_VISIBLE_CLEARED;
if (changingPart)
xlrec.flags |= XLH_DELETE_IS_PARTITION_MOVE;
@@ -3137,13 +3198,27 @@ heap_delete(Relation relation, ItemPointer tid,
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
+ if (vmbuffer_modified)
+ XLogRegisterBuffer(HEAP_DELETE_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE);
PageSetLSN(page, recptr);
+
+ if (vmbuffer_modified)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /*
+ * Release VM lock first, since it covers many heap blocks. We locked
+ * vmbuffer if clear_all_visible was true regardless of whether or not we
+ * ended up modifying the vmbuffer.
+ */
+ if (clear_all_visible)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
if (vmbuffer != InvalidBuffer)
@@ -3261,13 +3336,16 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
HeapTuple heaptup;
HeapTuple old_key_tuple = NULL;
bool old_key_copied = false;
- Page page;
+ Page page,
+ newpage;
BlockNumber block;
MultiXactStatus mxact_status;
Buffer buffer,
newbuf,
vmbuffer = InvalidBuffer,
vmbuffer_new = InvalidBuffer;
+ bool unlock_vmbuffer = false;
+ bool unlock_vmbuffer_new = false;
bool need_toast;
Size newtupsize,
pagefree;
@@ -3276,8 +3354,10 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
bool use_hot_update = false;
bool summarized_update = false;
bool key_intact;
- bool all_visible_cleared = false;
- bool all_visible_cleared_new = false;
+ bool clear_all_visible = false;
+ bool clear_all_visible_new = false;
+ bool vmbuffer_modified = false;
+ bool vmbuffer_new_modified = false;
bool checked_lockers;
bool locker_remains;
bool id_has_external = false;
@@ -3852,6 +3932,12 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
Assert(HEAP_XMAX_IS_LOCKED_ONLY(infomask_lock_old_tuple));
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuffer = true;
+ }
+
START_CRIT_SECTION();
/* Clear obsolete visibility flags ... */
@@ -3874,10 +3960,13 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
* overhead would be unchanged, that doesn't seem necessarily
* worthwhile.
*/
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ /* It's possible all-frozen was already clear */
+ if (visibilitymap_clear_locked(relation, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ }
MarkBufferDirty(buffer);
@@ -3896,12 +3985,24 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
xlrec.flags =
cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (unlock_vmbuffer)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ unlock_vmbuffer = false;
+
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
/*
@@ -3987,6 +4088,8 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
heaptup = newtup;
}
+ newpage = BufferGetPage(newbuf);
+
/*
* We're about to do the actual update -- check for conflict first, to
* avoid possibly having to roll back work we've just done.
@@ -4050,6 +4153,69 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
id_has_external,
&old_key_copied);
+ clear_all_visible = PageIsAllVisible(page);
+ clear_all_visible_new = newbuf != buffer && PageIsAllVisible(newpage);
+
+ /*
+ * Clear PD_ALL_VISIBLE flags and reset visibility map bits for any heap
+ * pages that were all-visible. If there are two heap pages, we may need
+ * to clear VM bits for both.
+ */
+ if (clear_all_visible && clear_all_visible_new &&
+ vmbuffer_new == vmbuffer)
+ {
+ /*
+ * This is the more complicated case: both the new and old heap pages
+ * are all-visible and both their VM bits are on the same page of the
+ * VM, so we register a single VM buffer as HEAP_UPDATE_BLKREF_VM_NEW
+ * in the WAL record. We must be careful to only lock and register one
+ * buffer, even though we modify it twice -- once for each heap
+ * block's VM bits.
+ */
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuffer_new = true;
+
+ /* We will not lock or attempt to modify old VM buffer */
+ }
+ else
+ {
+ /*
+ * In all the remaining cases, we will clear at most one heap block's
+ * VM bits per VM page.
+ */
+ Buffer vmbuffers[2] = {
+ clear_all_visible ? vmbuffer : InvalidBuffer,
+ clear_all_visible_new ? vmbuffer_new : InvalidBuffer
+ };
+
+ /*
+ * When both pages need different VM pages cleared, acquire the VM
+ * buffer locks in VM block order to avoid deadlocks between backends
+ * updating tuples in opposite directions across VM pages.
+ */
+ if (clear_all_visible && clear_all_visible_new &&
+ BufferGetBlockNumber(vmbuffers[0]) > BufferGetBlockNumber(vmbuffers[1]))
+ {
+ Buffer swap = vmbuffers[0];
+
+ vmbuffers[0] = vmbuffers[1];
+ vmbuffers[1] = swap;
+ }
+
+ Assert((!BufferIsValid(vmbuffers[0]) && !BufferIsValid(vmbuffers[1])) ||
+ vmbuffers[0] != vmbuffers[1]);
+
+ if (BufferIsValid(vmbuffers[0]))
+ LockBuffer(vmbuffers[0], BUFFER_LOCK_EXCLUSIVE);
+ if (BufferIsValid(vmbuffers[1]))
+ LockBuffer(vmbuffers[1], BUFFER_LOCK_EXCLUSIVE);
+
+ if (clear_all_visible)
+ unlock_vmbuffer = true;
+ if (clear_all_visible_new)
+ unlock_vmbuffer_new = true;
+ }
+
/* NO EREPORT(ERROR) from here till changes are logged */
START_CRIT_SECTION();
@@ -4086,7 +4252,6 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
RelationPutHeapTuple(relation, newbuf, heaptup, false); /* insert new tuple */
-
/* Clear obsolete visibility flags, possibly set by ourselves above... */
oldtup.t_data->t_infomask &= ~(HEAP_XMAX_BITS | HEAP_MOVED);
oldtup.t_data->t_infomask2 &= ~HEAP_KEYS_UPDATED;
@@ -4100,20 +4265,42 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
/* record address of new tuple in t_ctid of old one */
oldtup.t_data->t_ctid = heaptup->t_self;
- /* clear PD_ALL_VISIBLE flags, reset all visibilitymap bits */
- if (PageIsAllVisible(BufferGetPage(buffer)))
+ /*
+ * Clear PD_ALL_VISIBLE flags and reset all visibilitymap bits. In all
+ * cases, it's possible that PD_ALL_VISIBLE was set but the corresponding
+ * visibility map bits were already clear.
+ */
+ if (clear_all_visible)
{
- all_visible_cleared = true;
- PageClearAllVisible(BufferGetPage(buffer));
- visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
- vmbuffer, VISIBILITYMAP_VALID_BITS);
+ if (visibilitymap_clear_locked(relation, block,
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ {
+ /*
+ * When old and new heap blocks' VM bits are on the same VM page,
+ * that page is registered in the WAL record only once. If both
+ * heap pages were PD_ALL_VISIBLE and either VM bit needs
+ * clearing, we register the VM buffer as
+ * HEAP_UPDATE_BLKREF_VM_NEW.
+ */
+ if (clear_all_visible_new && vmbuffer == vmbuffer_new)
+ vmbuffer_new_modified = true;
+ else
+ vmbuffer_modified = true;
+ }
+
+ PageClearAllVisible(page);
}
- if (newbuf != buffer && PageIsAllVisible(BufferGetPage(newbuf)))
+ if (clear_all_visible_new)
{
- all_visible_cleared_new = true;
- PageClearAllVisible(BufferGetPage(newbuf));
- visibilitymap_clear(relation, BufferGetBlockNumber(newbuf),
- vmbuffer_new, VISIBILITYMAP_VALID_BITS);
+ /*
+ * If both heap blocks' VM bits are on the same VM buffer, this will
+ * clear the new heap block's VM bits from the shared vmbuffer.
+ */
+ if (visibilitymap_clear_locked(relation, BufferGetBlockNumber(newbuf),
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_new_modified = true;
+
+ PageClearAllVisible(newpage);
}
if (newbuf != buffer)
@@ -4136,19 +4323,30 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
}
recptr = log_heap_update(relation, buffer,
- newbuf, &oldtup, heaptup,
+ vmbuffer_modified ? vmbuffer : InvalidBuffer,
+ newbuf,
+ vmbuffer_new_modified ? vmbuffer_new : InvalidBuffer,
+ &oldtup, heaptup,
old_key_tuple,
- all_visible_cleared,
- all_visible_cleared_new);
+ clear_all_visible,
+ clear_all_visible_new);
if (newbuf != buffer)
- {
- PageSetLSN(BufferGetPage(newbuf), recptr);
- }
+ PageSetLSN(newpage, recptr);
PageSetLSN(BufferGetPage(buffer), recptr);
+
+ if (vmbuffer_modified)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
+ if (vmbuffer_new_modified)
+ PageSetLSN(BufferGetPage(vmbuffer_new), recptr);
}
END_CRIT_SECTION();
+ if (unlock_vmbuffer)
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ if (unlock_vmbuffer_new)
+ LockBuffer(vmbuffer_new, BUFFER_LOCK_UNLOCK);
+
if (newbuf != buffer)
LockBuffer(newbuf, BUFFER_LOCK_UNLOCK);
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
@@ -4586,6 +4784,7 @@ heap_lock_tuple(Relation relation, HeapTuple tuple,
ItemId lp;
Page page;
Buffer vmbuffer = InvalidBuffer;
+ bool unlock_vmbuffer = false;
BlockNumber block;
TransactionId xid,
xmax;
@@ -4599,6 +4798,7 @@ heap_lock_tuple(Relation relation, HeapTuple tuple,
*buffer = ReadBuffer(relation, ItemPointerGetBlockNumber(tid));
block = ItemPointerGetBlockNumber(tid);
+ page = BufferGetPage(*buffer);
/*
* Before locking the buffer, pin the visibility map page if it appears to
@@ -4606,12 +4806,11 @@ heap_lock_tuple(Relation relation, HeapTuple tuple,
* in the middle of changing this, so we'll need to recheck after we have
* the lock.
*/
- if (PageIsAllVisible(BufferGetPage(*buffer)))
+ if (PageIsAllVisible(page))
visibilitymap_pin(relation, block, &vmbuffer);
LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
- page = BufferGetPage(*buffer);
lp = PageGetItemId(page, ItemPointerGetOffsetNumber(tid));
Assert(ItemIdIsNormal(lp));
@@ -5166,6 +5365,13 @@ heap_lock_tuple(Relation relation, HeapTuple tuple,
GetCurrentTransactionId(), mode, false,
&xid, &new_infomask, &new_infomask2);
+ /* Lock VM buffer before entering critical section */
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuffer = true;
+ }
+
START_CRIT_SECTION();
/*
@@ -5197,11 +5403,13 @@ heap_lock_tuple(Relation relation, HeapTuple tuple,
tuple->t_data->t_ctid = *tid;
/* Clear only the all-frozen bit on visibility map if needed */
- if (PageIsAllVisible(page) &&
- visibilitymap_clear(relation, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
-
+ if (PageIsAllVisible(page))
+ {
+ /* It's possible all-frozen was already clear */
+ if (visibilitymap_clear_locked(relation, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ }
MarkBufferDirty(*buffer);
@@ -5232,19 +5440,33 @@ heap_lock_tuple(Relation relation, HeapTuple tuple,
xlrec.flags = cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
XLogRegisterData(&xlrec, SizeOfHeapLock);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
/* we don't decode row locks atm, so no need to log the origin */
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (unlock_vmbuffer)
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ unlock_vmbuffer = false;
+ }
+
result = TM_Ok;
out_locked:
LockBuffer(*buffer, BUFFER_LOCK_UNLOCK);
+ Assert(!unlock_vmbuffer);
out_unlocked:
if (BufferIsValid(vmbuffer))
@@ -5707,6 +5929,7 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
ItemPointerData tupid;
HeapTupleData mytup;
Buffer buf;
+ Page page;
uint16 new_infomask,
new_infomask2,
old_infomask,
@@ -5716,6 +5939,7 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
bool cleared_all_frozen = false;
bool pinned_desired_page;
Buffer vmbuffer = InvalidBuffer;
+ bool unlock_vmbuffer = false;
BlockNumber block;
ItemPointerCopy(tid, &tupid);
@@ -5724,6 +5948,7 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
{
new_infomask = 0;
new_xmax = InvalidTransactionId;
+ cleared_all_frozen = false;
block = ItemPointerGetBlockNumber(&tupid);
ItemPointerCopy(&tupid, &(mytup.t_self));
@@ -5743,13 +5968,15 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
l4:
CHECK_FOR_INTERRUPTS();
+ page = BufferGetPage(buf);
+
/*
* Before locking the buffer, pin the visibility map page if it
* appears to be necessary. Since we haven't got the lock yet,
* someone else might be in the middle of changing this, so we'll need
* to recheck after we have the lock.
*/
- if (PageIsAllVisible(BufferGetPage(buf)))
+ if (PageIsAllVisible(page))
{
visibilitymap_pin(rel, block, &vmbuffer);
pinned_desired_page = true;
@@ -5770,7 +5997,7 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
* this page. If this page isn't all-visible, we won't use the vm
* page, but we hold onto such a pin till the end of the function.
*/
- if (!pinned_desired_page && PageIsAllVisible(BufferGetPage(buf)))
+ if (!pinned_desired_page && PageIsAllVisible(page))
{
LockBuffer(buf, BUFFER_LOCK_UNLOCK);
visibilitymap_pin(rel, block, &vmbuffer);
@@ -5951,10 +6178,11 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
xid, mode, false,
&new_xmax, &new_infomask, &new_infomask2);
- if (PageIsAllVisible(BufferGetPage(buf)) &&
- visibilitymap_clear(rel, block, vmbuffer,
- VISIBILITYMAP_ALL_FROZEN))
- cleared_all_frozen = true;
+ if (PageIsAllVisible(page))
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ unlock_vmbuffer = true;
+ }
START_CRIT_SECTION();
@@ -5967,12 +6195,19 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
MarkBufferDirty(buf);
+ if (PageIsAllVisible(page))
+ {
+ /* It's possible all-frozen was already clear */
+ if (visibilitymap_clear_locked(rel, block, vmbuffer,
+ VISIBILITYMAP_ALL_FROZEN))
+ cleared_all_frozen = true;
+ }
+
/* XLOG stuff */
if (RelationNeedsWAL(rel))
{
xl_heap_lock_updated xlrec;
XLogRecPtr recptr;
- Page page = BufferGetPage(buf);
XLogBeginInsert();
XLogRegisterBuffer(HEAP_LOCK_BLKREF_HEAP, buf, REGBUF_STANDARD);
@@ -5985,13 +6220,26 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
XLogRegisterData(&xlrec, SizeOfHeapLockUpdated);
+ if (cleared_all_frozen)
+ XLogRegisterBuffer(HEAP_LOCK_BLKREF_VM, vmbuffer, 0);
+
recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_LOCK_UPDATED);
PageSetLSN(page, recptr);
+
+ if (cleared_all_frozen)
+ PageSetLSN(BufferGetPage(vmbuffer), recptr);
}
END_CRIT_SECTION();
+ /* release VM lock first, since it covers many heap blocks */
+ if (unlock_vmbuffer)
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);
+ unlock_vmbuffer = false;
+ }
+
next:
/* if we find the end of update chain, we're done. */
if (mytup.t_data->t_infomask & HEAP_XMAX_INVALID ||
@@ -6017,6 +6265,7 @@ heap_lock_updated_tuple_rec(Relation rel, TransactionId priorXmax,
out_unlocked:
if (vmbuffer != InvalidBuffer)
ReleaseBuffer(vmbuffer);
+ Assert(!unlock_vmbuffer);
return result;
}
@@ -8848,8 +9097,9 @@ log_heap_visible(Relation rel, Buffer heap_buffer, Buffer vm_buffer,
* have modified the buffer(s) and marked them dirty.
*/
static XLogRecPtr
-log_heap_update(Relation reln, Buffer oldbuf,
- Buffer newbuf, HeapTuple oldtup, HeapTuple newtup,
+log_heap_update(Relation reln, Buffer oldbuf, Buffer vmbuffer_old,
+ Buffer newbuf, Buffer vmbuffer_new,
+ HeapTuple oldtup, HeapTuple newtup,
HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared)
{
@@ -9058,6 +9308,20 @@ log_heap_update(Relation reln, Buffer oldbuf,
old_key_tuple->t_len - SizeofHeapTupleHeader);
}
+ /*
+ * Register VM buffers. If the old and new heap pages' VM bits are on the
+ * same VM page, the caller passes only vmbuffer_new (mirroring the heap
+ * page convention where block 0 = new is always registered).
+ */
+ Assert((BufferIsInvalid(vmbuffer_old) && BufferIsInvalid(vmbuffer_new)) ||
+ (vmbuffer_old != vmbuffer_new));
+
+ if (BufferIsValid(vmbuffer_new))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_NEW, vmbuffer_new, 0);
+
+ if (BufferIsValid(vmbuffer_old))
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_OLD, vmbuffer_old, 0);
+
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index 87b5c44fd46..0ce23dedfe4 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -22,6 +22,70 @@
#include "storage/freespace.h"
#include "storage/standby.h"
+/*
+ * Clear visibility map bits for a single heap block during heap redo.
+ *
+ * Used by records that modify one heap block and, at most, its corresponding
+ * VM block (insert, delete, multi_insert, lock). Records that can touch
+ * multiple heap or VM blocks (e.g. updates) replay the VM changes inline
+ * instead.
+ *
+ * 'record' is the WAL record being replayed
+ * 'target_locator' identifies the relation whose VM is being updated
+ * 'heap_blkno' is the heap block whose VM bits should be cleared
+ * 'wal_vm_block_id' is the WAL block reference id of the VM page
+ * 'flags' specifies which visibility map bits to clear
+ */
+static void
+heap_xlog_vm_clear(XLogReaderState *record,
+ RelFileLocator target_locator,
+ BlockNumber heap_blkno,
+ uint8 wal_vm_block_id, uint8 flags)
+{
+ XLogRecPtr lsn = record->EndRecPtr;
+ Relation reln = CreateFakeRelcacheEntry(target_locator);
+ Buffer vmbuffer = InvalidBuffer;
+
+ /*
+ * If the vmbuffer was registered, use the recovery-specific routines to
+ * read it. These will either apply an FPI or indicate that we should
+ * clear the requested bits ourselves.
+ *
+ * Originally, clearing the VM did not register the VM buffers, so since
+ * registering the VM buffer was a bug fix, we keep a fallback path to
+ * support replay of WAL generated from before the fix.
+ */
+ if (XLogRecHasBlockRef(record, wal_vm_block_id))
+ {
+ if (XLogReadBufferForRedo(record, wal_vm_block_id,
+ &vmbuffer) == BLK_NEEDS_REDO)
+ {
+ if (visibilitymap_clear_locked(reln,
+ heap_blkno, vmbuffer,
+ flags))
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ }
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else
+ {
+ /*
+ * This is the backwards compatibility path to clear VM bits for
+ * records predating VM buffer registration. It is also invoked if the
+ * heap page's PD_ALL_VISIBLE was cleared but the VM bits were already
+ * clear. The WAL record flags do not distinguish between these two
+ * situations. Though this is wasted effort, the behavior is
+ * historical and the situation should be rare.
+ */
+ visibilitymap_pin(reln, heap_blkno, &vmbuffer);
+ visibilitymap_clear(reln, heap_blkno, vmbuffer, flags);
+ ReleaseBuffer(vmbuffer);
+ }
+
+ FreeFakeRelcacheEntry(reln);
+}
+
/*
* Replay XLOG_HEAP2_PRUNE_* records.
@@ -360,15 +424,9 @@ heap_xlog_delete(XLogReaderState *record)
* already up-to-date.
*/
if (xlrec->flags & XLH_DELETE_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(target_locator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
+ heap_xlog_vm_clear(record, target_locator,
+ blkno, HEAP_DELETE_BLKREF_VM,
+ VISIBILITYMAP_VALID_BITS);
if (XLogReadBufferForRedo(record, HEAP_DELETE_BLKREF_HEAP,
&buffer) == BLK_NEEDS_REDO)
@@ -449,15 +507,9 @@ heap_xlog_insert(XLogReaderState *record)
* already up-to-date.
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(target_locator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
+ heap_xlog_vm_clear(record, target_locator,
+ blkno, HEAP_INSERT_BLKREF_VM,
+ VISIBILITYMAP_VALID_BITS);
/*
* If we inserted the first and only tuple on the page, re-initialize the
@@ -573,17 +625,15 @@ heap_xlog_multi_insert(XLogReaderState *record)
/*
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
+ *
+ * Clear the VM (if needed) before clearing the heap page-level visibility
+ * flag (PD_ALL_VISIBLE) to prevent the heap page from being marked
+ * all-visible in the VM while its PD_ALL_VISIBLE is clear.
*/
if (xlrec->flags & XLH_INSERT_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, blkno, &vmbuffer);
- visibilitymap_clear(reln, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
+ heap_xlog_vm_clear(record, rlocator,
+ blkno, HEAP_MULTI_INSERT_BLKREF_VM,
+ VISIBILITYMAP_VALID_BITS);
if (isinit)
{
@@ -698,6 +748,8 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Buffer obuffer,
nbuffer;
Page page;
+ bool new_cleared,
+ old_cleared;
OffsetNumber offnum;
ItemId lp = NULL;
HeapTupleData oldtup;
@@ -737,14 +789,95 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
*/
- if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED)
+ new_cleared = (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED) != 0;
+ old_cleared = (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) != 0;
+ if (new_cleared || old_cleared)
{
Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
+ bool has_vm_old = XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_OLD);
+ bool has_vm_new = XLogRecHasBlockRef(record, HEAP_UPDATE_BLKREF_VM_NEW);
+
+ if (has_vm_new)
+ {
+ Buffer vmbuffer_new = InvalidBuffer;
+
+ Assert(xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED);
+
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW,
+ &vmbuffer_new) == BLK_NEEDS_REDO)
+ {
+ /*
+ * If both the old and new heap pages were all-visible and
+ * their VM bits are on the same VM page, that single VM page
+ * is registered as HEAP_UPDATE_BLKREF_VM_NEW. Clear both heap
+ * blocks' VM bits from the single provided VM buffer. It's
+ * possible that one of the page's VM bits were already clear,
+ * but visibilitymap_clear() is harmless as long as we provide
+ * it the correct bits.
+ *
+ * We must verify that oldblk's VM bits really are on this VM
+ * page, rather than relying on the absence of a separate
+ * VM_OLD block reference: VM_OLD is also omitted when oldblk
+ * is on a different VM page but its bit was already clear.
+ */
+ if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED &&
+ visibilitymap_pin_ok(oldblk, vmbuffer_new))
+ {
+ if (visibilitymap_clear_locked(reln, oldblk, vmbuffer_new,
+ VISIBILITYMAP_VALID_BITS))
+ PageSetLSN(BufferGetPage(vmbuffer_new), lsn);
+ }
+ /* If VM_NEW is registered, we are sure newblk is on VM_NEW */
+ if (visibilitymap_clear_locked(reln, newblk, vmbuffer_new,
+ VISIBILITYMAP_VALID_BITS))
+ PageSetLSN(BufferGetPage(vmbuffer_new), lsn);
+ }
+ if (BufferIsValid(vmbuffer_new))
+ UnlockReleaseBuffer(vmbuffer_new);
+ }
+ if (has_vm_old)
+ {
+ Buffer vmbuffer_old = InvalidBuffer;
+
+ Assert(xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED);
+
+ if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_OLD, &vmbuffer_old) ==
+ BLK_NEEDS_REDO)
+ {
+ if (visibilitymap_clear_locked(reln, oldblk, vmbuffer_old,
+ VISIBILITYMAP_VALID_BITS))
+ PageSetLSN(BufferGetPage(vmbuffer_old), lsn);
+ }
+ if (BufferIsValid(vmbuffer_old))
+ UnlockReleaseBuffer(vmbuffer_old);
+ }
+ if (!has_vm_old && !has_vm_new)
+ {
+ /*
+ * Backwards compatibility path. Previously, the VM buffers were
+ * not registered in the WAL record. We need this path to replay
+ * WAL generated by a not-yet-patched primary during upgrade.
+ */
+ if (old_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ visibilitymap_pin(reln, oldblk, &vmbuffer);
+ visibilitymap_clear(reln, oldblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
+ if (new_cleared)
+ {
+ Buffer vmbuffer = InvalidBuffer;
+
+ visibilitymap_pin(reln, newblk, &vmbuffer);
+ visibilitymap_clear(reln, newblk, vmbuffer,
+ VISIBILITYMAP_VALID_BITS);
+ ReleaseBuffer(vmbuffer);
+ }
+ }
- visibilitymap_pin(reln, oldblk, &vmbuffer);
- visibilitymap_clear(reln, oldblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
@@ -752,10 +885,12 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
* In normal operation, it is important to lock the two pages in
* page-number order, to avoid possible deadlocks against other update
* operations going the other way. However, during WAL replay there can
- * be no other update happening, so we don't need to worry about that. But
- * we *do* need to worry that we don't expose an inconsistent state to Hot
- * Standby queries --- so the original page can't be unlocked before we've
- * added the new tuple to the new page.
+ * be no other update happening, so we don't need to worry about that.
+ * Notice we also don't worry about this when locking VM buffers above.
+ *
+ * But we *do* need to worry that we don't expose an inconsistent state to
+ * Hot Standby queries --- so the original page can't be unlocked before
+ * we've added the new tuple to the new page.
*/
/* Deal with old tuple version */
@@ -793,7 +928,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
/* Mark the page as a candidate for pruning */
PageSetPrunable(page, XLogRecGetXid(record));
- if (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED)
+ if (old_cleared)
PageClearAllVisible(page);
PageSetLSN(page, lsn);
@@ -819,21 +954,6 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
newaction = XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_HEAP_NEW,
&nbuffer);
- /*
- * The visibility map may need to be fixed even if the heap page is
- * already up-to-date.
- */
- if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED)
- {
- Relation reln = CreateFakeRelcacheEntry(rlocator);
- Buffer vmbuffer = InvalidBuffer;
-
- visibilitymap_pin(reln, newblk, &vmbuffer);
- visibilitymap_clear(reln, newblk, vmbuffer, VISIBILITYMAP_VALID_BITS);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
- }
-
/* Deal with new tuple */
if (newaction == BLK_NEEDS_REDO)
{
@@ -930,7 +1050,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
if (offnum == InvalidOffsetNumber)
elog(PANIC, "failed to add tuple");
- if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED)
+ if (new_cleared)
PageClearAllVisible(page);
freespace = PageGetHeapFreeSpace(page); /* needed to update FSM below */
@@ -1022,20 +1142,15 @@ heap_xlog_lock(XLogReaderState *record)
*/
if (xlrec->flags & XLH_LOCK_ALL_FROZEN_CLEARED)
{
- RelFileLocator rlocator;
- Buffer vmbuffer = InvalidBuffer;
BlockNumber block;
- Relation reln;
+ RelFileLocator rlocator;
XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
&block);
- reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ heap_xlog_vm_clear(record, rlocator,
+ block, HEAP_LOCK_BLKREF_VM,
+ VISIBILITYMAP_ALL_FROZEN);
}
if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
@@ -1100,20 +1215,15 @@ heap_xlog_lock_updated(XLogReaderState *record)
*/
if (xlrec->flags & XLH_LOCK_ALL_FROZEN_CLEARED)
{
- RelFileLocator rlocator;
- Buffer vmbuffer = InvalidBuffer;
BlockNumber block;
- Relation reln;
+ RelFileLocator rlocator;
XLogRecGetBlockTag(record, HEAP_LOCK_BLKREF_HEAP, &rlocator, NULL,
&block);
- reln = CreateFakeRelcacheEntry(rlocator);
-
- visibilitymap_pin(reln, block, &vmbuffer);
- visibilitymap_clear(reln, block, vmbuffer, VISIBILITYMAP_ALL_FROZEN);
- ReleaseBuffer(vmbuffer);
- FreeFakeRelcacheEntry(reln);
+ heap_xlog_vm_clear(record, rlocator,
+ block, HEAP_LOCK_BLKREF_VM,
+ VISIBILITYMAP_ALL_FROZEN);
}
if (XLogReadBufferForRedo(record, HEAP_LOCK_BLKREF_HEAP,
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index 1874a3fda37..d44620eca7f 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -135,9 +135,38 @@ static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks);
* You must pass a buffer containing the correct map page to this function.
* Call visibilitymap_pin first to pin the right one. This function doesn't do
* any I/O. Returns true if any bits have been cleared and false otherwise.
+ *
+ * Most callers should use visibilitymap_clear_locked rather than this
+ * function. It is usually necessary to register the VM buffer in the WAL
+ * record, and this necessitates holding the lock for longer than it is held
+ * here. However, we retain this function for behavioral compatibility on the
+ * back branches (out-of-tree callers may expect it to take the lock). And,
+ * since we have it, we use it for in-tree callers that don't have to manage
+ * the VM buffer lock themselves.
*/
bool
visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
+{
+ bool cleared = false;
+
+ LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+
+ cleared = visibilitymap_clear_locked(rel, heapBlk, vmbuf, flags);
+
+ LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
+
+ return cleared;
+}
+
+/*
+ * Clear specified bits, caller holds VM lock
+ *
+ * Like visibilitymap_clear(), except the caller must already hold the VM
+ * buffer exclusive lock and is responsible for unlocking it.
+ * Returns true if any bits were actually cleared, false otherwise.
+ */
+bool
+visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
{
BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
@@ -157,7 +186,8 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
elog(ERROR, "wrong buffer passed to visibilitymap_clear");
- LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
+ Assert(BufferIsExclusiveLocked(vmbuf));
+
map = PageGetContents(BufferGetPage(vmbuf));
if (map[mapByte] & mask)
@@ -168,8 +198,6 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
cleared = true;
}
- LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
-
return cleared;
}
diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl
index 0f98c7df82e..fcb701e1b98 100644
--- a/src/bin/pg_walsummary/t/002_blocks.pl
+++ b/src/bin/pg_walsummary/t/002_blocks.pl
@@ -93,13 +93,14 @@ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
split(m@/@, $end_lsn);
ok(-f $filename, "WAL summary file exists");
-# Run pg_walsummary on it. We expect exactly two blocks to be modified,
-# block 0 and one other.
+# Run pg_walsummary on it. We expect exactly three blocks to be modified,
+# block 0 (old tuple), another block (new tuple), and the block for the VM.
my ($stdout, $stderr) = run_command([ 'pg_walsummary', '-i', $filename ]);
note($stdout);
@lines = split(/\n/, $stdout);
like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified");
+like($stdout, qr/FORK vm: block 0$/m, "stdout shows VM block 0 modified");
is($stderr, '', 'stderr is empty');
-is(0 + @lines, 2, "UPDATE modified 2 blocks");
+is(0 + @lines, 3, "UPDATE modified 3 blocks");
done_testing();
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 92744c53899..c4c3c3f15fa 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -111,6 +111,7 @@
/* This is what we need to know about delete */
#define HEAP_DELETE_BLKREF_HEAP 0
+#define HEAP_DELETE_BLKREF_VM 1
typedef struct xl_heap_delete
{
@@ -160,6 +161,7 @@ typedef struct xl_heap_header
/* This is what we need to know about insert */
#define HEAP_INSERT_BLKREF_HEAP 0
+#define HEAP_INSERT_BLKREF_VM 1
typedef struct xl_heap_insert
{
@@ -179,6 +181,7 @@ typedef struct xl_heap_insert
* (XLOG_HEAP_INIT_PAGE).
*/
#define HEAP_MULTI_INSERT_BLKREF_HEAP 0
+#define HEAP_MULTI_INSERT_BLKREF_VM 1
/*
* In HEAP_MULTI_INSERT_BLKREF_HEAP's data portion, there is an
@@ -222,10 +225,21 @@ typedef struct xl_multi_insert_tuple
*
* HEAP_UPDATE_BLKREF_HEAP_OLD: old page, if different. (no data, just a reference
* to the block)
+ *
+ * HEAP_UPDATE_BLKREF_VM_NEW: VM page covering the new heap page. Registered
+ * when XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED is set and the new heap page's VM
+ * bit was actually cleared. Also covers the old heap page's VM bits when both
+ * heap pages map to the same VM page.
+ *
+ * HEAP_UPDATE_BLKREF_VM_OLD: VM page covering the old heap page. Only
+ * registered when XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED is set, the old heap
+ * page's VM bits are on a different VM page from the new heap page's, and the
+ * old heap page's VM bit was actually cleared.
*/
-
#define HEAP_UPDATE_BLKREF_HEAP_NEW 0
#define HEAP_UPDATE_BLKREF_HEAP_OLD 1
+#define HEAP_UPDATE_BLKREF_VM_NEW 2
+#define HEAP_UPDATE_BLKREF_VM_OLD 3
typedef struct xl_heap_update
{
@@ -406,6 +420,7 @@ typedef struct xlhp_prune_items
/* This is what we need to know about lock */
#define HEAP_LOCK_BLKREF_HEAP 0
+#define HEAP_LOCK_BLKREF_VM 1
typedef struct xl_heap_lock
{
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index ea889bf9ec7..1bc59c9ac33 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -26,6 +26,8 @@
#define VM_ALL_FROZEN(r, b, v) \
((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0)
+extern bool visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk,
+ Buffer vmbuf, uint8 flags);
extern bool visibilitymap_clear(Relation rel, BlockNumber heapBlk,
Buffer vmbuf, uint8 flags);
extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk,
--
2.47.3
v5-0003-Put-pg_combinebackup-debug-test-output-in-temp-fi.patch (text/x-patch, 4.2K) download
From 2cbd9da9b46f7541032dd000dfb9013fe91ed730 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 30 Apr 2026 16:06:06 -0400
Subject: [PATCH v5 3/4] Put pg_combinebackup debug test output in temp file
Having --debug as the default option to pg_combinebackup in
init_from_backup() makes the logs extremely verbose. It is used for
debugging failures, so put the output in a temp file and only keep it
around if tests fail.
Author: Melanie Plageman <melanieplageman@gmail.com>
Disussion: https://postgr.es/m/y73coty2smkxgvelh4e32kqrt5pyn2pb7cz7x4mrz6yaxitqd5%40nzhkeazqf4bn
Backpatch-through: 17
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 83 +++++++++++++++++++++++-
1 file changed, 81 insertions(+), 2 deletions(-)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 2e31c23a6ac..3388e45b726 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -125,6 +125,8 @@ our $min_compat = 12;
# list of file reservations made by get_free_port
my @port_reservation_files;
+my @combinebackup_debug_logs;
+my %combinebackup_debug_logs_printed;
# We want to choose a server port above the range that servers typically use
# on Unix systems and below the range those systems typically use for ephemeral
@@ -898,7 +900,9 @@ pathnames that should be used for the target cluster.
To restore from an incremental backup, pass the parameter combine_with_prior
as a reference to an array of prior backup names with which this backup
-is to be combined using pg_combinebackup.
+is to be combined using pg_combinebackup. pg_combinebackup is run with
+--debug, but the output is captured in a temporary file and only copied to the
+test log if pg_combinebackup fails or if a later test fails.
Streaming replication can be enabled on this node by passing the keyword
parameter has_streaming => 1. This is disabled by default.
@@ -964,7 +968,7 @@ sub init_from_backup
}
push @combineargs, @prior_backup_path, $backup_path,
'--output' => $data_path;
- PostgreSQL::Test::Utils::system_or_bail(@combineargs);
+ _run_pg_combinebackup_with_debug_log(@combineargs);
}
elsif (defined $params{tar_program})
{
@@ -1079,6 +1083,81 @@ port = $port
return;
}
+sub _run_pg_combinebackup_with_debug_log
+{
+ my (@cmd) = @_;
+ my $debug_log = PostgreSQL::Test::Utils::tempdir('pg_combinebackup') .
+ '/debug.log';
+
+ push @combinebackup_debug_logs, $debug_log;
+
+ print("# Running: " . join(" ", @cmd) . "\n");
+ # If this pg_combinebackup invocation itself fails, print its captured
+ # output immediately. Successful invocations may still be printed later if
+ # some subsequent TAP assertion in this test script fails.
+ if (!IPC::Run::run(\@cmd, '&>' => $debug_log))
+ {
+ _print_pg_combinebackup_debug_log($debug_log);
+
+ if ($? == -1)
+ {
+ BAIL_OUT(
+ sprintf(
+ "failed to execute command \"%s\": $!", join(" ", @cmd)));
+ }
+ elsif ($? & 127)
+ {
+ BAIL_OUT(
+ sprintf(
+ "command \"%s\" died with signal %d",
+ join(" ", @cmd), $? & 127));
+ }
+ else
+ {
+ BAIL_OUT(
+ sprintf(
+ "command \"%s\" exited with value %d",
+ join(" ", @cmd), $? >> 8));
+ }
+ }
+}
+
+sub _print_pg_combinebackup_debug_log
+{
+ my ($debug_log) = @_;
+
+ return unless -e $debug_log;
+ return if $combinebackup_debug_logs_printed{$debug_log}++;
+
+ diag("pg_combinebackup debug output from $debug_log:\n" .
+ PostgreSQL::Test::Utils::slurp_file($debug_log));
+}
+
+# Keep pg_combinebackup debug logs until test exit. If pg_combinebackup itself
+# fails, its log is printed at the failure site above. If pg_combinebackup
+# succeeds but a later TAP assertion fails, print the retained logs here so the
+# regress output includes the combine step that produced the restored state.
+# Remove the logs after a clean run.
+END
+{
+ # take care not to change the script's exit value
+ my $exit_code = $?;
+
+ if ($exit_code == 0 && PostgreSQL::Test::Utils::all_tests_passing())
+ {
+ unlink @combinebackup_debug_logs;
+ }
+ else
+ {
+ foreach my $debug_log (@combinebackup_debug_logs)
+ {
+ _print_pg_combinebackup_debug_log($debug_log);
+ }
+ }
+
+ $? = $exit_code;
+}
+
=pod
=item $node->rotate_logfile()
--
2.47.3
v5-0004-Test-that-VM-clear-registers-VM-buffers.patch (text/x-patch, 10.1K) download
From 414cf30429bc77429a26e267906f56b6a51cc463 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 23 Apr 2026 11:13:23 -0400
Subject: [PATCH v5 4/4] Test that VM clear registers VM buffers
The WAL summarizer only tracks registered buffers, so unregistered VM
clears are ommitted from incremental backups, corrupting the restored
visibility map. Test those cases are now fixed.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Andrey Borodin <x4mmm@yandex-team.ru>
Discussion: https://postgr.es/m/oqcsevg35xjan2327x5kdfth6q4fgeqboxfo3v3imeyih2uiny%406sez5dzxl6nt
Backpatch-through: 17
---
src/bin/pg_combinebackup/Makefile | 2 +
src/bin/pg_combinebackup/meson.build | 1 +
.../pg_combinebackup/t/012_vm_consistency.pl | 258 ++++++++++++++++++
3 files changed, 261 insertions(+)
create mode 100644 src/bin/pg_combinebackup/t/012_vm_consistency.pl
diff --git a/src/bin/pg_combinebackup/Makefile b/src/bin/pg_combinebackup/Makefile
index 33a1f4483bf..bdba0353412 100644
--- a/src/bin/pg_combinebackup/Makefile
+++ b/src/bin/pg_combinebackup/Makefile
@@ -12,6 +12,8 @@
PGFILEDESC = "pg_combinebackup - combine incremental backups"
PGAPPICON=win32
+EXTRA_INSTALL=contrib/pg_visibility
+
subdir = src/bin/pg_combinebackup
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build
index bbc4c5735ba..757b78e2fe8 100644
--- a/src/bin/pg_combinebackup/meson.build
+++ b/src/bin/pg_combinebackup/meson.build
@@ -39,6 +39,7 @@ tests += {
't/009_no_full_file.pl',
't/010_hardlink.pl',
't/011_ib_truncation.pl',
+ 't/012_vm_consistency.pl',
],
}
}
diff --git a/src/bin/pg_combinebackup/t/012_vm_consistency.pl b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
new file mode 100644
index 00000000000..b0810f8022a
--- /dev/null
+++ b/src/bin/pg_combinebackup/t/012_vm_consistency.pl
@@ -0,0 +1,258 @@
+# Copyright (c) 2026, PostgreSQL Global Development Group
+#
+# Test that heap operations clearing visibility map bits (INSERT, UPDATE,
+# DELETE, SELECT FOR UPDATE, COPY) correctly register visibility map buffers,
+# since incremental backups rely on the WAL summarizer, which only tracks
+# registered buffers.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir_short();
+my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy';
+
+# Set up primary with WAL summarization enabled.
+my $primary = PostgreSQL::Test::Cluster->new('primary');
+$primary->init(allows_streaming => 1);
+$primary->append_conf('postgresql.conf', <<EOF);
+summarize_wal = on
+autovacuum = off
+EOF
+$primary->start;
+
+$primary->safe_psql('postgres', q{CREATE EXTENSION pg_visibility});
+
+my @tests = (
+ {
+ label => 'INSERT',
+ table => 'vm_insert_test',
+ setup => q{CREATE TABLE vm_insert_test (id int);
+ INSERT INTO vm_insert_test DEFAULT VALUES;},
+ modify => q{INSERT INTO vm_insert_test VALUES (1)},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'DELETE',
+ table => 'vm_delete_test',
+ setup => q{CREATE TABLE vm_delete_test (id int);
+ INSERT INTO vm_delete_test VALUES (1), (2);},
+ modify => q{DELETE FROM vm_delete_test WHERE id = 1},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'UPDATE',
+ table => 'vm_update_test',
+ # Include both same-page and cross-page updates. val is stored PLAIN
+ # so the large update stays inline.
+ setup => q{CREATE TABLE vm_update_test (id INT, val TEXT);
+ ALTER TABLE vm_update_test ALTER COLUMN val SET STORAGE PLAIN;
+ INSERT INTO vm_update_test VALUES (1, 'same page'), (2, 'cross page');
+ INSERT INTO vm_update_test SELECT i, repeat('a', 200)
+ FROM generate_series(3, 70) i;},
+ modify => q{UPDATE vm_update_test SET id = 0 WHERE id = 1;
+ UPDATE vm_update_test SET val = repeat('b', 4000) WHERE id = 2;},
+ # Confirm the small update stays on the same heap page while the large
+ # update relocates the tuple to a different heap page.
+ ctid_checks => [
+ {
+ before => 'id = 1',
+ after => 'id = 0',
+ same => 1,
+ desc => 'small update stays on the same heap page',
+ },
+ {
+ before => 'id = 2',
+ after => 'id = 2',
+ same => 0,
+ desc => 'large update moves the tuple to a different heap page',
+ },
+ ],
+ visible_op => '<',
+ frozen_op => '<',
+ },
+ {
+ label => 'LOCK',
+ table => 'vm_lock_test',
+ setup => q{CREATE TABLE vm_lock_test (id int);
+ INSERT INTO vm_lock_test VALUES (1), (2);},
+ modify => q{SELECT * FROM vm_lock_test WHERE id = 1 FOR UPDATE},
+ visible_op => '==',
+ frozen_op => '<',
+ },
+ {
+ label => 'COPY',
+ table => 'vm_copy_test',
+ setup => q{CREATE TABLE vm_copy_test (id int);
+ INSERT INTO vm_copy_test DEFAULT VALUES;},
+ modify => q{COPY vm_copy_test FROM PROGRAM 'echo 42'},
+ visible_op => '<',
+ frozen_op => '<',
+ },
+);
+
+sub get_vm_summary
+{
+ my ($node, $table) = @_;
+ my $result = $node->safe_psql('postgres',
+ "SELECT all_visible, all_frozen FROM pg_visibility_map_summary('$table')");
+ my @vals = split(/\|/, $result);
+ return @vals;
+}
+
+# Return the heap block number of the (single) row matching $where. The ctid
+# is "(block,offset)"; casting it through point lets us pull out the block.
+sub heap_block
+{
+ my ($node, $table, $where) = @_;
+ return $node->safe_psql('postgres',
+ "SELECT (ctid::text::point)[0]::int FROM $table WHERE $where");
+}
+
+# Confirm VACUUM (FREEZE) set VM bits before testing whether later heap
+# modifications clear those bits and are captured by incremental backup. We
+# could perhaps be more exact than > 0, but the coarseness attempts to avoid
+# tset flakes.
+sub check_vacuumed_vm
+{
+ my ($node, $test) = @_;
+ my ($all_visible, $all_frozen) = get_vm_summary($node, $test->{table});
+
+ cmp_ok($all_visible, '>', 0,
+ "$test->{label} test: pages are all-visible after vacuum");
+ cmp_ok($all_frozen, '>', 0,
+ "$test->{label} test: pages are all-frozen after vacuum");
+
+ return ($all_visible, $all_frozen);
+}
+
+# Check the VM bit counts after a heap modification against the post-vacuum
+# baseline. Most operations clear both bits; tuple locking clears all-frozen
+# without clearing all-visible.
+sub check_modified_vm
+{
+ my ($node, $test) = @_;
+ my ($post_visible, $post_frozen) = get_vm_summary($node, $test->{table});
+
+ cmp_ok($post_visible, $test->{visible_op}, $test->{pre_visible},
+ "$test->{label} test: all-visible state after modification");
+ cmp_ok($post_frozen, $test->{frozen_op}, $test->{pre_frozen},
+ "$test->{label} test: all-frozen state after modification");
+
+ return ($post_visible, $post_frozen);
+}
+
+# Verify the combined backup restored VM state exactly as it exists on the
+# primary, and ask pg_visibility to check that visible tuples are consistent.
+sub validate_restored_vm
+{
+ my ($restored, $test) = @_;
+
+ my ($primary_visible, $primary_frozen) =
+ get_vm_summary($primary, $test->{table});
+ my ($restored_visible, $restored_frozen) =
+ get_vm_summary($restored, $test->{table});
+
+ is($restored_visible, $primary_visible,
+ "$test->{label} test: restored all_visible count matches primary");
+ is($restored_frozen, $primary_frozen,
+ "$test->{label} test: restored all_frozen count matches primary");
+
+ my $corrupt_tids = $restored->safe_psql('postgres',
+ "SELECT count(*) FROM pg_check_visible('$test->{table}')");
+ is($corrupt_tids, '0',
+ "$test->{label} test: no VM corruption detected by pg_check_visible");
+}
+
+# Create and populate the tables, then vacuum freeze them to set the VM bits
+foreach my $test (@tests)
+{
+ $primary->safe_psql('postgres', $test->{setup});
+ $primary->safe_psql('postgres', "VACUUM (FREEZE) $test->{table}");
+ ($test->{pre_visible}, $test->{pre_frozen}) =
+ check_vacuumed_vm($primary, $test);
+}
+
+# Take a full backup
+my $full_name = 'full';
+my $full_path = $primary->backup_dir . "/$full_name";
+$primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $full_path,
+ '--checkpoint' => 'fast',
+ ],
+ 'full backup');
+
+# Modify the tables and check that the VM bits are as expected for that test
+# after the specified modification.
+foreach my $test (@tests)
+{
+ # Record the heap block of any rows whose same-/cross-page movement we
+ # want to verify, before the modification relocates them.
+ foreach my $check (@{ $test->{ctid_checks} // [] })
+ {
+ $check->{before_blk} =
+ heap_block($primary, $test->{table}, $check->{before});
+ }
+
+ $primary->safe_psql('postgres', $test->{modify});
+ ($test->{post_visible}, $test->{post_frozen}) =
+ check_modified_vm($primary, $test);
+
+ # Confirm each checked row did (or did not) move to a different heap page.
+ foreach my $check (@{ $test->{ctid_checks} // [] })
+ {
+ my $after_blk = heap_block($primary, $test->{table}, $check->{after});
+ if ($check->{same})
+ {
+ is($after_blk, $check->{before_blk},
+ "$test->{label} test: $check->{desc}");
+ }
+ else
+ {
+ isnt($after_blk, $check->{before_blk},
+ "$test->{label} test: $check->{desc}");
+ }
+ }
+}
+
+# Take an incremental backup. This will have the changes made in the
+# modification step.
+my $incr_name = 'incr';
+my $incr_path = $primary->backup_dir . "/$incr_name";
+$primary->command_ok(
+ [
+ 'pg_basebackup', '--no-sync',
+ '--pgdata' => $incr_path,
+ '--checkpoint' => 'fast',
+ '--incremental' => $full_path . '/backup_manifest',
+ ],
+ 'incremental backup');
+
+# Start a server from a combined backup composed of the incremental and full
+# backup.
+my $restored = PostgreSQL::Test::Cluster->new('restored');
+$restored->init_from_backup($primary, $incr_name,
+ combine_with_prior => [$full_name],
+ combine_mode => $mode);
+$restored->append_conf('postgresql.conf', <<EOF);
+autovacuum = off
+EOF
+$restored->start;
+$restored->safe_psql('postgres', q{CREATE EXTENSION IF NOT EXISTS pg_visibility});
+
+# Confirm that the restored server's visibility map matches the original server
+foreach my $test (@tests)
+{
+ validate_restored_vm($restored, $test);
+}
+
+$restored->stop;
+$primary->stop;
+
+done_testing();
--
2.47.3
heap_update.diff (text/x-patch, 2.8K) download
From 10739902aaafa214e4a9df161bea280a7139d34b Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Tue, 14 Jul 2026 14:24:16 -0400
Subject: [PATCH] heap_update
---
src/backend/access/heap/heapam.c | 57 ++++++++++++++++++--------------
1 file changed, 32 insertions(+), 25 deletions(-)
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index fd8ca72728f..e445ea35096 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -4270,38 +4270,45 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
* cases, it's possible that PD_ALL_VISIBLE was set but the corresponding
* visibility map bits were already clear.
*/
- if (clear_all_visible)
- {
- if (visibilitymap_clear_locked(relation, block,
- vmbuffer, VISIBILITYMAP_VALID_BITS))
- {
- /*
- * When old and new heap blocks' VM bits are on the same VM page,
- * that page is registered in the WAL record only once. If both
- * heap pages were PD_ALL_VISIBLE and either VM bit needs
- * clearing, we register the VM buffer as
- * HEAP_UPDATE_BLKREF_VM_NEW.
- */
- if (clear_all_visible_new && vmbuffer == vmbuffer_new)
- vmbuffer_new_modified = true;
- else
- vmbuffer_modified = true;
- }
-
- PageClearAllVisible(page);
- }
- if (clear_all_visible_new)
+ if (clear_all_visible && clear_all_visible_new &&
+ vmbuffer_new == vmbuffer)
{
/*
- * If both heap blocks' VM bits are on the same VM buffer, this will
- * clear the new heap block's VM bits from the shared vmbuffer.
+ * When old and new heap blocks' VM bits are on the same VM page, that
+ * page is registered in the WAL record only once. If both heap pages
+ * were PD_ALL_VISIBLE and either VM bit needs clearing, we register
+ * the VM buffer as HEAP_UPDATE_BLKREF_VM_NEW.
*/
- if (visibilitymap_clear_locked(relation, BufferGetBlockNumber(newbuf),
- vmbuffer_new, VISIBILITYMAP_VALID_BITS))
+ if (visibilitymap_clear(relation, block,
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_new_modified = true;
+
+ if (visibilitymap_clear(relation, BufferGetBlockNumber(newbuf),
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS))
vmbuffer_new_modified = true;
+ PageClearAllVisible(page);
PageClearAllVisible(newpage);
}
+ else
+ {
+ if (clear_all_visible)
+ {
+ if (visibilitymap_clear(relation, block,
+ vmbuffer, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_modified = true;
+
+ PageClearAllVisible(page);
+ }
+ if (clear_all_visible_new)
+ {
+ if (visibilitymap_clear(relation, BufferGetBlockNumber(newbuf),
+ vmbuffer_new, VISIBILITYMAP_VALID_BITS))
+ vmbuffer_new_modified = true;
+
+ PageClearAllVisible(newpage);
+ }
+ }
if (newbuf != buffer)
MarkBufferDirty(newbuf);
--
2.47.3
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-15 18:27 Robert Haas <robertmhaas@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 1 reply; 57+ messages in thread
From: Robert Haas @ 2026-07-15 18:27 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Fri, Jul 10, 2026 at 3:07 PM Melanie Plageman
<melanieplageman@gmail.com> wrote:
> _run_pg_combinebackup_with_debug_log's error handling is basically
> copied from system_or_bail -- is that too duplicative? Also, I'm not
> sure if the END code block is done correctly or not.
I don't think that part is too duplicative.
I am not a huge fan of relying on an END block here -- global
destructors can be finnicky to work with, and while passing critical
information around in random global variables is a time-honored
PostgreSQL tradition, it's not a fantastic practice, either.
I think we might want to consider stashing the logs in $self and then
let the caller decide whether to call $self->print_deferred_logs.
Tests that rely on this functionality could choose a place at which to
call $self->print_deferred_logs if !
PostgreSQL::Test::Utils::all_tests_passing(). As compared with that
approach, nothing would be printed if we bail out, unless the caller
arranges to catch the bail out and print the logs before we actually
die, but maybe that's fine.
Or maybe we could simplify this by not even saving all the logs and
just printing the whole log if it fails, else not. Or print the whole
log if it fails and use _diag_command_output() if it succeeds?
(Granted, that's "private" right now, but we could rename it to
diag_command_output and document it.) Or maybe always use
_diag_command_output() and if a future failure needs something from
the middle of the output then we'll deal with that problem then? I'm
not sure what the chances are that we're going to need this output in
the future -- it depends on how many bugs there are in the incremental
backup code, and knowing that number is difficult. I'm hesitant to
just discard it, but overcomplicating the code isn't great, either.
+ my $debug_log = PostgreSQL::Test::Utils::tempdir('pg_combinebackup') .
+ '/debug.log';
I think this is making a separate temporary directory for each
invocation of pg_combinebackup, each containing one file. Perhaps
that's a little more profligate than we need to be.
> Also combinebackup_debug_logs_printed hash set is a little
> over-complicated -- it was added because we could print the same log
> output twice if pg_combinebackup itself failed. Maybe there is a less
> complicated way to handle this? I don't think the END block is
> guaranteed to run depending on how pg_combinebackup fails, so I don't
> know that we can be sure the logs print without doing it right when
> pg_combinebackup fails.
I think the problem here might just that you do push
@combinebackup_debug_logs, $debug_log; before running the command, so
that if the command fails it prints and then the global destructor
sees $debug_log in the list and prints again. You could presumably fix
this by either moving this push down to the end of the function, or by
having _run_pg_combinebackup_with_debug_log not also call
_print_pg_combinebackup_debug_log and rely entirely on END.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-15 19:34 Melanie Plageman <melanieplageman@gmail.com>
parent: Robert Haas <robertmhaas@gmail.com>
0 siblings, 1 reply; 57+ messages in thread
From: Melanie Plageman @ 2026-07-15 19:34 UTC (permalink / raw)
To: Robert Haas <robertmhaas@gmail.com>; +Cc: Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
Thanks for the review. Reflecting on it more, it probably makes more
sense to commit the VM clear fix and propose pg_combinebackup test
verbosity in a separate thread targeted at master. My current test
version only takes one incremental backup, so it shouldn't be too much
of an issue.
I've addressed your feedback below inline and attached the current
version of the patch you reviewed (with feedback addressed), so that
when I link to this thread in the new thread I plan to make, it all
makes sense.
On Wed, Jul 15, 2026 at 2:27 PM Robert Haas <robertmhaas@gmail.com> wrote:
>
> On Fri, Jul 10, 2026 at 3:07 PM Melanie Plageman
> <melanieplageman@gmail.com> wrote:
> > _run_pg_combinebackup_with_debug_log's error handling is basically
> > copied from system_or_bail -- is that too duplicative? Also, I'm not
> > sure if the END code block is done correctly or not.
>
> I don't think that part is too duplicative.
>
> I am not a huge fan of relying on an END block here -- global
> destructors can be finnicky to work with, and while passing critical
> information around in random global variables is a time-honored
> PostgreSQL tradition, it's not a fantastic practice, either.
>
> I think we might want to consider stashing the logs in $self and then
> let the caller decide whether to call $self->print_deferred_logs.
> Tests that rely on this functionality could choose a place at which to
> call $self->print_deferred_logs if !
> PostgreSQL::Test::Utils::all_tests_passing(). As compared with that
> approach, nothing would be printed if we bail out, unless the caller
> arranges to catch the bail out and print the logs before we actually
> die, but maybe that's fine.
>
> Or maybe we could simplify this by not even saving all the logs and
> just printing the whole log if it fails, else not. Or print the whole
> log if it fails and use _diag_command_output() if it succeeds?
> (Granted, that's "private" right now, but we could rename it to
> diag_command_output and document it.) Or maybe always use
> _diag_command_output() and if a future failure needs something from
> the middle of the output then we'll deal with that problem then? I'm
> not sure what the chances are that we're going to need this output in
> the future -- it depends on how many bugs there are in the incremental
> backup code, and knowing that number is difficult. I'm hesitant to
> just discard it, but overcomplicating the code isn't great, either.
>
> + my $debug_log = PostgreSQL::Test::Utils::tempdir('pg_combinebackup') .
> + '/debug.log';
I do now save the logs in $self, but the rest of these thoughts seem
like they would be best discussed in a new thread, so I'll move it
there.
> I think this is making a separate temporary directory for each
> invocation of pg_combinebackup, each containing one file. Perhaps
> that's a little more profligate than we need to be.
Instead, I just avoided creating a temp directory and put the log
files in the node's base directory. It's still cleaned up with the
node, and still kept on failure (basedir honors PG_TEST_NOCLEAN /
test-failure retention).
> > Also combinebackup_debug_logs_printed hash set is a little
> > over-complicated -- it was added because we could print the same log
> > output twice if pg_combinebackup itself failed. Maybe there is a less
> > complicated way to handle this? I don't think the END block is
> > guaranteed to run depending on how pg_combinebackup fails, so I don't
> > know that we can be sure the logs print without doing it right when
> > pg_combinebackup fails.
>
> I think the problem here might just that you do push
> @combinebackup_debug_logs, $debug_log; before running the command, so
> that if the command fails it prints and then the global destructor
> sees $debug_log in the list and prints again. You could presumably fix
> this by either moving this push down to the end of the function, or by
> having _run_pg_combinebackup_with_debug_log not also call
> _print_pg_combinebackup_debug_log and rely entirely on END.
Got it. For now I moved the push to the end of the function (after the
if (!IPC::Run::run…) block). Then if pg_combinebackup fails, it prints
right away but if it succeeds and a later assertion fails, those print
at the end. I think END won't run if perl crashes -- maybe I don't
need to solve for that, though?
- Melanie
Attachments:
Put-pg_combinebackup-debug-test-output-in-temp-file.patch (text/x-patch, 4.5K) download
From 67ff8ceca9a4b49de87bbb9f5e13dabefe0e6253 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 30 Apr 2026 16:06:06 -0400
Subject: [PATCH] Put pg_combinebackup debug test output in temp file
Having --debug as the default option to pg_combinebackup in
init_from_backup() makes the logs extremely verbose. It is used for
debugging failures, so put the output in a temp file and only keep it
around if tests fail.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Robert Haas <robertmhaas@gmail.com>
Disussion: https://postgr.es/m/y73coty2smkxgvelh4e32kqrt5pyn2pb7cz7x4mrz6yaxitqd5%40nzhkeazqf4bn
Backpatch-through: 17
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 86 +++++++++++++++++++++++-
1 file changed, 84 insertions(+), 2 deletions(-)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 529f49efee1..0d4fd929b6b 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -125,6 +125,7 @@ our $min_compat = 12;
# list of file reservations made by get_free_port
my @port_reservation_files;
+my @combinebackup_debug_logs;
# We want to choose a server port above the range that servers typically use
# on Unix systems and below the range those systems typically use for ephemeral
@@ -924,7 +925,9 @@ pathnames that should be used for the target cluster.
To restore from an incremental backup, pass the parameter combine_with_prior
as a reference to an array of prior backup names with which this backup
-is to be combined using pg_combinebackup.
+is to be combined using pg_combinebackup. pg_combinebackup is run with
+--debug, but the output is captured in a temporary file and only copied to the
+test log if pg_combinebackup fails or if a later test fails.
Streaming replication can be enabled on this node by passing the keyword
parameter has_streaming => 1. This is disabled by default.
@@ -990,7 +993,7 @@ sub init_from_backup
}
push @combineargs, @prior_backup_path, $backup_path,
'--output' => $data_path;
- PostgreSQL::Test::Utils::system_or_bail(@combineargs);
+ $self->_run_pg_combinebackup_with_debug_log(@combineargs);
}
elsif (defined $params{tar_program})
{
@@ -1105,6 +1108,85 @@ port = $port
return;
}
+sub _run_pg_combinebackup_with_debug_log
+{
+ my ($self, @cmd) = @_;
+ my $debug_log = $self->basedir . '/pg_combinebackup_debug.log';
+
+ print("# Running: " . join(" ", @cmd) . "\n");
+ # If this pg_combinebackup invocation itself fails, print its captured
+ # output immediately. Successful invocations may still be printed later if
+ # some subsequent TAP assertion in this test script fails.
+ if (!IPC::Run::run(\@cmd, '&>' => $debug_log))
+ {
+ _print_pg_combinebackup_debug_log($debug_log);
+
+ if ($? == -1)
+ {
+ BAIL_OUT(
+ sprintf(
+ "failed to execute command \"%s\": $!", join(" ", @cmd)));
+ }
+ elsif ($? & 127)
+ {
+ BAIL_OUT(
+ sprintf(
+ "command \"%s\" died with signal %d",
+ join(" ", @cmd), $? & 127));
+ }
+ else
+ {
+ BAIL_OUT(
+ sprintf(
+ "command \"%s\" exited with value %d",
+ join(" ", @cmd), $? >> 8));
+ }
+ }
+
+ # pg_combinebackup succeeded. Retain its debug log so that, if a later TAP
+ # assertion in this test fails, the END block can print the combine step
+ # that produced the restored state; on a clean run the END block removes it.
+ # On failure above we already printed the log and bailed out, so it is
+ # deliberately not added to the list here (which also avoids printing it
+ # twice).
+ push @combinebackup_debug_logs, $debug_log;
+}
+
+sub _print_pg_combinebackup_debug_log
+{
+ my ($debug_log) = @_;
+
+ return unless -e $debug_log;
+
+ diag("pg_combinebackup debug output from $debug_log:\n" .
+ PostgreSQL::Test::Utils::slurp_file($debug_log));
+}
+
+# Keep pg_combinebackup debug logs until test exit. If pg_combinebackup itself
+# fails, its log is printed at the failure site above. If pg_combinebackup
+# succeeds but a later TAP assertion fails, print the retained logs here so the
+# regress output includes the combine step that produced the restored state.
+# Remove the logs after a clean run.
+END
+{
+ # take care not to change the script's exit value
+ my $exit_code = $?;
+
+ if ($exit_code == 0 && PostgreSQL::Test::Utils::all_tests_passing())
+ {
+ unlink @combinebackup_debug_logs;
+ }
+ else
+ {
+ foreach my $debug_log (@combinebackup_debug_logs)
+ {
+ _print_pg_combinebackup_debug_log($debug_log);
+ }
+ }
+
+ $? = $exit_code;
+}
+
=pod
=item $node->rotate_logfile()
--
2.47.3
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-15 20:10 Robert Haas <robertmhaas@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
0 siblings, 0 replies; 57+ messages in thread
From: Robert Haas @ 2026-07-15 20:10 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Wed, Jul 15, 2026 at 3:34 PM Melanie Plageman
<melanieplageman@gmail.com> wrote:
> Got it. For now I moved the push to the end of the function (after the
> if (!IPC::Run::run…) block). Then if pg_combinebackup fails, it prints
> right away but if it succeeds and a later assertion fails, those print
> at the end. I think END won't run if perl crashes -- maybe I don't
> need to solve for that, though?
If perl dumps core, that's not your problem.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-15 21:58 Melanie Plageman <melanieplageman@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
0 siblings, 1 reply; 57+ messages in thread
From: Melanie Plageman @ 2026-07-15 21:58 UTC (permalink / raw)
To: Robert Haas <robertmhaas@gmail.com>; +Cc: Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Tue, Jul 14, 2026 at 3:42 PM Melanie Plageman
<melanieplageman@gmail.com> wrote:
>
> I've also finished all branch versions of this. Each one's code
> differs, but I've managed to keep the commit messages the same, which
> should help release notes. Note that REL_19_STABLE bumps
> XLOG_PAGE_MAGIC. This would keep us from having to maintain the
> backwards compatibility code there, which I think is what we want.
> Here are those branches.
I've pushed the main commits to all branches where needed. There are
three patches remaining that I proposed for master that do assorted
validations: visibility map and PD_ALL_VISIBLE flag validations and
some validation in the WAL machinery that registered buffers get used.
I plan to commit 0001 and 0003 once my other patches have gone through
the buildfarm. I'm considering holding onto 0002 until I add
WAL-logging to the VM corruption repair code in
heap_page_fix_vm_corruption() -- without which,
wal_consistency_checking could fail for pages that had corruption
fixed.
- Melanie
Attachments: download all (3 files, .tar.gz)
v6-0001-Add-VM-corruption-check-to-verify_heapam.patch (application/x-patch, 2.2K) download
From 17584bc5632da492b7a01a55993ae78d61d846e0 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 10:20:20 -0400
Subject: [PATCH v6 1/3] Add VM corruption check to verify_heapam()
pg_check_visible() can catch cases were tuples are not visible to all
and the VM is set, but it doesn't specifically check page hints. This
doesn't check, for example, an INSERT failing to clear PD_ALL_VISIBLE.
This also helps users who have access to amcheck but not pg_visibility
to diagnose corruption.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/flat/CAAKRu_bn%2Be7F4yPFBgFbnP%2BsyJRKyNK092bjD2LKvZW7O4Svag%40mail.gmail.com
---
contrib/amcheck/verify_heapam.c | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 20ff58aa782..823f05e5916 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -481,6 +481,7 @@ verify_heapam(PG_FUNCTION_ARGS)
while ((ctx.buffer = read_stream_next_buffer(stream, NULL)) != InvalidBuffer)
{
+ uint8 vmbits;
OffsetNumber maxoff;
OffsetNumber predecessor[MaxOffsetNumber];
OffsetNumber successor[MaxOffsetNumber];
@@ -499,6 +500,24 @@ verify_heapam(PG_FUNCTION_ARGS)
ctx.blkno = BufferGetBlockNumber(ctx.buffer);
ctx.page = BufferGetPage(ctx.buffer);
+ /*
+ * It is corruption if the page-level PD_ALL_VISIBLE flag is clear and
+ * the visibility map bits corresponding to this heap page are set.
+ * Any corruption on the VM page itself won't be caught, and won't
+ * cause visibilitymap_get_status() to fail, because the VM page is
+ * read with ZERO_ON_ERROR.
+ */
+ vmbits = visibilitymap_get_status(ctx.rel, ctx.blkno, &vmbuffer);
+
+ if (!PageIsAllVisible(ctx.page) &&
+ (vmbits & VISIBILITYMAP_VALID_BITS) != 0)
+ {
+ ctx.offnum = InvalidOffsetNumber;
+ ctx.attnum = -1;
+ report_corruption(&ctx,
+ psprintf("page is not marked all-visible in page header but visibility map bit is set"));
+ }
+
/* Perform tuple checks */
maxoff = PageGetMaxOffsetNumber(ctx.page);
for (ctx.offnum = FirstOffsetNumber; ctx.offnum <= maxoff;
--
2.47.3
v6-0002-Stop-masking-PD_ALL_VISIBLE-when-checking-WAL-con.patch (application/x-patch, 1.6K) download
From 357ea85a4decebafaab750e7812f01ef90db6fb2 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 10:33:46 -0400
Subject: [PATCH v6 2/3] Stop masking PD_ALL_VISIBLE when checking WAL
consistency
Setting and clearing PD_ALL_VISIBLE is WAL-logged, so there is no need
to mask it for the purposes of WAL consistency checking.
There is one minor exception, which is that when fixing VM corruption,
we clear PD_ALL_VISIBLE on the heap page without specifically emitting
WAL. This is a situation where corruption has already happened, however,
so it is okay if WAL consistency checking flags this as well.
Stop masking PD_ALL_VISIBLE, as doing so can only conceal real bugs.
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/flat/CAAKRu_bn%2Be7F4yPFBgFbnP%2BsyJRKyNK092bjD2LKvZW7O4Svag%40mail.gmail.com
---
src/backend/access/common/bufmask.c | 6 ------
1 file changed, 6 deletions(-)
diff --git a/src/backend/access/common/bufmask.c b/src/backend/access/common/bufmask.c
index 5f63d04c9cb..93e34ea38b8 100644
--- a/src/backend/access/common/bufmask.c
+++ b/src/backend/access/common/bufmask.c
@@ -53,12 +53,6 @@ mask_page_hint_bits(Page page)
/* Ignore PD_PAGE_FULL and PD_HAS_FREE_LINES flags, they are just hints. */
PageClearFull(page);
PageClearHasFreeLinePointers(page);
-
- /*
- * PD_ALL_VISIBLE is masked during WAL consistency checking. XXX: It is
- * worth investigating if we could stop doing this.
- */
- PageClearAllVisible(page);
}
/*
--
2.47.3
v6-0003-xlog-Verify-all-block-references-are-used-during-.patch (application/x-patch, 4.2K) download
From 2d81d0d6194fb6df98f37f624943e80e37a27fc4 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 29 Apr 2026 12:08:26 -0400
Subject: [PATCH v6 3/3] xlog: Verify all block references are used during
replay
It's unclear if this makes sense to actually commit. It requires that
blocks read during recovery go through XLogReadBufferForRedoExtended()
and has a special case for handling XLOG_FPI_FOR_HINT when full page
writes are disabled.
Author: Andres Freund <andres@anarazel.de>
Co-authored-by: Melanie Plageman <melanieplageman@gmail.com>
Discussion: https://postgr.es/m/oqcsevg35xjan2327x5kdfth6q4fgeqboxfo3v3imeyih2uiny%406sez5dzxl6nt
---
src/backend/access/transam/xlog.c | 10 ++++++++++
src/backend/access/transam/xlogreader.c | 4 ++++
src/backend/access/transam/xlogrecovery.c | 16 ++++++++++++++++
src/backend/access/transam/xlogutils.c | 6 ++++++
src/include/access/xlogreader.h | 4 ++++
5 files changed, 40 insertions(+)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f8b939853e9..40ba0ac7279 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9084,6 +9084,16 @@ xlog_redo(XLogReaderState *record)
{
if (info == XLOG_FPI)
elog(ERROR, "XLOG_FPI record did not contain a full-page image");
+
+#ifdef USE_ASSERT_CHECKING
+
+ /*
+ * If full_page_writes are disabled, we don't want to error
+ * out because we didn't read the block.
+ */
+ if (info == XLOG_FPI_FOR_HINT)
+ record->record->blocks[block_id].used_read = true;
+#endif
continue;
}
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 946907a2507..8679a2435ed 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1796,6 +1796,10 @@ DecodeXLogRecord(XLogReaderState *state,
blk = &decoded->blocks[block_id];
blk->in_use = true;
+#ifdef USE_ASSERT_CHECKING
+ blk->used_read = false;
+#endif
+
blk->apply_image = false;
COPY_HEADER_FIELD(&fork_flags, sizeof(uint8));
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index a9ebac2d0ef..a5dfbfbdcef 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -1965,6 +1965,22 @@ ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *repl
/* Now apply the WAL record itself */
GetRmgr(record->xl_rmid).rm_redo(xlogreader);
+ /*
+ * Verify that all block references were used during replay. This helps
+ * detect bugs in redo routines. Every referenced block needs to be
+ * replayed with XLogReadBufferForRedoExtended(), possibly via a helper,
+ * to ensure that we replay FPIs, extend the relation if necessary, and
+ * other perform other similar protections.
+ */
+#ifdef USE_ASSERT_CHECKING
+ for (int block_id = 0; block_id <= xlogreader->record->max_block_id; block_id++)
+ {
+ DecodedBkpBlock *blk = &xlogreader->record->blocks[block_id];
+
+ Assert(!blk->in_use || blk->used_read);
+ }
+#endif
+
/*
* After redo, check whether the backup pages associated with the WAL
* record are consistent with the existing pages. This check is done only
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 58b9dab6a90..cc7c8a06089 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -381,6 +381,12 @@ XLogReadBufferForRedoExtended(XLogReaderState *record,
block_id);
}
+#ifdef USE_ASSERT_CHECKING
+ /* Shouldn't have multiple read references to block */
+ Assert(!record->record->blocks[block_id].used_read);
+ record->record->blocks[block_id].used_read = true;
+#endif
+
/*
* Make sure that if the block is marked with WILL_INIT, the caller is
* going to initialize it. And vice versa.
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 4a9a687e879..5b598b20859 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -145,6 +145,10 @@ typedef struct
bool has_data;
char *data;
uint16 data_len;
+
+#ifdef USE_ASSERT_CHECKING
+ bool used_read;
+#endif
} DecodedBkpBlock;
/*
--
2.47.3
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-16 01:30 Fujii Masao <masao.fujii@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
0 siblings, 1 reply; 57+ messages in thread
From: Fujii Masao @ 2026-07-16 01:30 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: Robert Haas <robertmhaas@gmail.com>; Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Thu, Jul 16, 2026 at 6:58 AM Melanie Plageman
<melanieplageman@gmail.com> wrote:
>
> On Tue, Jul 14, 2026 at 3:42 PM Melanie Plageman
> <melanieplageman@gmail.com> wrote:
> >
> > I've also finished all branch versions of this. Each one's code
> > differs, but I've managed to keep the commit messages the same, which
> > should help release notes. Note that REL_19_STABLE bumps
> > XLOG_PAGE_MAGIC. This would keep us from having to maintain the
> > backwards compatibility code there, which I think is what we want.
> > Here are those branches.
>
> I've pushed the main commits to all branches where needed.
Thanks for working on them!
In commit ed62d26caca
- if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
- elog(ERROR, "wrong buffer passed to visibilitymap_clear");
+ Assert(BufferIsValid(vmbuf) && BufferGetBlockNumber(vmbuf) == mapBlock);
Isn't it be better to keep the elog(ERROR) in addition to the Assert()
so that wrong buffer is detected even in non-assert builds?
Otherwise, a non-assert build could silently modify the wrong VM page
if a caller passes the wrong buffer. Since this commit increased the number
of callers and introduced visibilitymap_clear() into WAL redo paths,
it seems safer to retain the runtime check.
Regards,
--
Fujii Masao
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-16 22:35 Melanie Plageman <melanieplageman@gmail.com>
parent: Fujii Masao <masao.fujii@gmail.com>
0 siblings, 2 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-07-16 22:35 UTC (permalink / raw)
To: Fujii Masao <masao.fujii@gmail.com>; +Cc: Robert Haas <robertmhaas@gmail.com>; Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Wed, Jul 15, 2026 at 9:31 PM Fujii Masao <masao.fujii@gmail.com> wrote:
>
> Thanks for working on them!
Thanks!
> In commit ed62d26caca
>
> - if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
> - elog(ERROR, "wrong buffer passed to visibilitymap_clear");
> + Assert(BufferIsValid(vmbuf) && BufferGetBlockNumber(vmbuf) == mapBlock);
>
> Isn't it be better to keep the elog(ERROR) in addition to the Assert()
> so that wrong buffer is detected even in non-assert builds?
>
> Otherwise, a non-assert build could silently modify the wrong VM page
> if a caller passes the wrong buffer. Since this commit increased the number
> of callers and introduced visibilitymap_clear() into WAL redo paths,
> it seems safer to retain the runtime check.
I had a vague memory of discussing this with someone at some point
(and that person agreeing we should turn the error into an assert) but
now I can't find any record of it. My thought was that only developers
should be introducing new callers of visibilitymap_clear() and
developers should be using assert builds. My understanding is
typically we stick with asserts for situations we don't think users
can get themselves and would only be caused by wayward developers.
However, visibilitymap_set() still has the error, so it's probably
better to be consistent. And there's no reason to change this in
backbranches at least. I'll change it back. It may take me a few days,
but I won't forget!
- Melanie
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-17 20:51 Melanie Plageman <melanieplageman@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 2 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-07-17 20:51 UTC (permalink / raw)
To: Fujii Masao <masao.fujii@gmail.com>; +Cc: Robert Haas <robertmhaas@gmail.com>; Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Thu, Jul 16, 2026 at 6:35 PM Melanie Plageman
<melanieplageman@gmail.com> wrote:
>
> I'll change it back. It may take me a few days,
> but I won't forget!
I've committed this.
- Melanie
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-17 21:18 Andres Freund <andres@anarazel.de>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 0 replies; 57+ messages in thread
From: Andres Freund @ 2026-07-17 21:18 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: Fujii Masao <masao.fujii@gmail.com>; Robert Haas <robertmhaas@gmail.com>; Andrey Borodin <x4mmm@yandex-team.ru>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
Hi,
On 2026-07-16 18:35:30 -0400, Melanie Plageman wrote:
> On Wed, Jul 15, 2026 at 9:31 PM Fujii Masao <masao.fujii@gmail.com> wrote:
> > - if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
> > - elog(ERROR, "wrong buffer passed to visibilitymap_clear");
> > + Assert(BufferIsValid(vmbuf) && BufferGetBlockNumber(vmbuf) == mapBlock);
> >
> > Isn't it be better to keep the elog(ERROR) in addition to the Assert()
> > so that wrong buffer is detected even in non-assert builds?
> >
> > Otherwise, a non-assert build could silently modify the wrong VM page
> > if a caller passes the wrong buffer. Since this commit increased the number
> > of callers and introduced visibilitymap_clear() into WAL redo paths,
> > it seems safer to retain the runtime check.
>
> I had a vague memory of discussing this with someone at some point
> (and that person agreeing we should turn the error into an assert) but
> now I can't find any record of it.
That might have been me.
I don't see any reason for this to be an elog(). So much would have to go
wrong for that to happen in a non-assert build. And we check many more crucial
things only in Asserts.
I'm also don't care too much about it being an elog, so I'm not going to push
for reverting the s/Assert/elog/. But I don't think it should be seen any sort
of policy about this kind of check needing to be an elog going forward.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-20 02:34 Tom Lane <tgl@sss.pgh.pa.us>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 1 reply; 57+ messages in thread
From: Tom Lane @ 2026-07-20 02:34 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: Fujii Masao <masao.fujii@gmail.com>; Robert Haas <robertmhaas@gmail.com>; Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
Melanie Plageman <melanieplageman@gmail.com> writes:
> I've committed this.
Coverity complained about this patch:
/srv/coverity/git/pgsql-git/postgresql/src/backend/access/heap/pruneheap.c: 951 in heap_page_fix_vm_corruption()
945 MarkBufferDirtyHint(prstate->buffer, true);
946 }
947
948 if (do_clear_vm)
949 {
950 LockBuffer(prstate->vmbuffer, BUFFER_LOCK_EXCLUSIVE);
>>> CID 1697133: Error handling issues (CHECKED_RETURN)
>>> Calling "visibilitymap_clear" without checking return value (as is done elsewhere 13 out of 14 times).
951 visibilitymap_clear(prstate->relation->rd_locator, prstate->block,
952 prstate->vmbuffer,
953 VISIBILITYMAP_VALID_BITS);
954 LockBuffer(prstate->vmbuffer, BUFFER_LOCK_UNLOCK);
955 prstate->old_vmbits = 0;
956 }
I think it's right to complain --- if we check for failure everywhere
else, why's it OK to not check here? If it is OK, a comment and an
explicit cast to "(void)" would be appropriate.
regards, tom lane
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-07-20 14:56 Melanie Plageman <melanieplageman@gmail.com>
parent: Tom Lane <tgl@sss.pgh.pa.us>
0 siblings, 1 reply; 57+ messages in thread
From: Melanie Plageman @ 2026-07-20 14:56 UTC (permalink / raw)
To: Tom Lane <tgl@sss.pgh.pa.us>; +Cc: Fujii Masao <masao.fujii@gmail.com>; Robert Haas <robertmhaas@gmail.com>; Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Sun, Jul 19, 2026 at 10:34 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>
> Melanie Plageman <melanieplageman@gmail.com> writes:
> > I've committed this.
>
> Coverity complained about this patch:
>
> /srv/coverity/git/pgsql-git/postgresql/src/backend/access/heap/pruneheap.c: 951 in heap_page_fix_vm_corruption()
> 945 MarkBufferDirtyHint(prstate->buffer, true);
> 946 }
> 947
> 948 if (do_clear_vm)
> 949 {
> 950 LockBuffer(prstate->vmbuffer, BUFFER_LOCK_EXCLUSIVE);
> >>> CID 1697133: Error handling issues (CHECKED_RETURN)
> >>> Calling "visibilitymap_clear" without checking return value (as is done elsewhere 13 out of 14 times).
> 951 visibilitymap_clear(prstate->relation->rd_locator, prstate->block,
> 952 prstate->vmbuffer,
> 953 VISIBILITYMAP_VALID_BITS);
> 954 LockBuffer(prstate->vmbuffer, BUFFER_LOCK_UNLOCK);
> 955 prstate->old_vmbits = 0;
> 956 }
>
> I think it's right to complain --- if we check for failure everywhere
> else, why's it OK to not check here? If it is OK, a comment and an
> explicit cast to "(void)" would be appropriate.
Got it. I didn't know we always checked return values everywhere in
the code base.
In other cases where we call visibilitymap_clear(), we use the return
value to decide whether or not to emit WAL (because if the page wasn't
modified, we shouldn't). Not emitting WAL here is its own questionable
issue (historical, not something I introduced), but since we don't, I
didn't need the return value. I'll push a comment and cast next week
(I'm traveling this week and don't commit on the road if I can help
it).
- Melanie
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-08-04 22:13 Melanie Plageman <melanieplageman@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
0 siblings, 0 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-08-04 22:13 UTC (permalink / raw)
To: Tom Lane <tgl@sss.pgh.pa.us>; +Cc: Fujii Masao <masao.fujii@gmail.com>; Robert Haas <robertmhaas@gmail.com>; Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Mon, Jul 20, 2026 at 10:56 AM Melanie Plageman
<melanieplageman@gmail.com> wrote:
>
> On Sun, Jul 19, 2026 at 10:34 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
> >
> > Melanie Plageman <melanieplageman@gmail.com> writes:
> > > I've committed this.
> >
> > Coverity complained about this patch:
> >
> > /srv/coverity/git/pgsql-git/postgresql/src/backend/access/heap/pruneheap.c: 951 in heap_page_fix_vm_corruption()
> > 945 MarkBufferDirtyHint(prstate->buffer, true);
> > 946 }
> > 947
> > 948 if (do_clear_vm)
> > 949 {
> > 950 LockBuffer(prstate->vmbuffer, BUFFER_LOCK_EXCLUSIVE);
> > >>> CID 1697133: Error handling issues (CHECKED_RETURN)
> > >>> Calling "visibilitymap_clear" without checking return value (as is done elsewhere 13 out of 14 times).
> > 951 visibilitymap_clear(prstate->relation->rd_locator, prstate->block,
> > 952 prstate->vmbuffer,
> > 953 VISIBILITYMAP_VALID_BITS);
> > 954 LockBuffer(prstate->vmbuffer, BUFFER_LOCK_UNLOCK);
> > 955 prstate->old_vmbits = 0;
> > 956 }
> >
> > I think it's right to complain --- if we check for failure everywhere
> > else, why's it OK to not check here? If it is OK, a comment and an
> > explicit cast to "(void)" would be appropriate.
>
> Got it. I didn't know we always checked return values everywhere in
> the code base.
>
> In other cases where we call visibilitymap_clear(), we use the return
> value to decide whether or not to emit WAL (because if the page wasn't
> modified, we shouldn't). Not emitting WAL here is its own questionable
> issue (historical, not something I introduced), but since we don't, I
> didn't need the return value. I'll push a comment and cast next week
> (I'm traveling this week and don't commit on the road if I can help
> it).
K, I committed this to 19 and master. In backbranches we actually
check the return value even less -- coverity was complaining because
we started checking the return value more often and it went over some
threshold.
- Melanie
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-09-16 16:10 Melanie Plageman <melanieplageman@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
2 siblings, 2 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-09-16 16:10 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; +Cc: Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
On Thu, Apr 30, 2026 at 5:44 PM Melanie Plageman
<melanieplageman@gmail.com> wrote:
>
> Attached is a patch set to fix the issue based largely on the work you
> started on your branch. I attached the version targeting master/19
> which is prefixed with v1_PGMASTER and the version targeting 18,
> prefixed v1_PG18. The pg 18 changes aren't a straight cherry-pick to
> 17 (the earliest I'll backpatch because that was when incremental
> backup was introduced) because the redo functions live in a different
> file in 18 than in 17, but I want to avoid discussing three different
> versions of this patch set on this thread.
>
> The backpatched changes are different for a few reasons, but the
> biggest difference from a review standpoint is that in pg18, the redo
> routines can read WAL in the old format or the new format, so that
> people can reasonably upgrade to the new minor version.
There is an interaction between CREATE DATABASE STRATEGY WAL_LOG and
this fix that causes data corruption (identified while investigating
[1]). The scenario is you have a heap page set PD_ALL_VISIBLE and a
corresponding VM page set all-visible, you do a CREATE DATABASE
STRATEGY WAL_LOG using that database as a template. The primary has
the heap page set PDV (PD_ALL_VISIBLE) set and VM page AV set but the
standby does NOT have the page set all-visible in the VM (it does have
the heap page set PDV). After failover you have a primary with heap
page PDV set and VM page not set. You modify a tuple on the page.
Because the VM was already clear, we do not register the VM block in
the WAL record. The standby replays this and clears the heap page
PD_ALL_VISIBLE but leaves the VM set. This is data corruption leading
to wrong results from index scans and worse if that standby is later
promoted back to primary.
We didn't have this problem before ed62d26caca because it cleared the
VM during replay as long as PD_ALL_VISIBLE needed clearing. I mention
in the thread [2] that I think we have to fix
RelationCopyStorageUsingBuffer() to avoid producing this scenario
because tuple locking already caused data corruption before
ed62d26caca. However, for update/delete, etc, I'm wondering if we
should harden this a bit and go back to registering a vm clear
operation whenever PD_ALL_VISIBLE was set on the heap page. If this is
a no-op on the primary for the VM, we register it with
REBGUF_NO_CHANGE. Vacuum works very hard to never end up with
PD_ALL_VISIBLE cleared and the VM set. Perhaps we should do everything
we can to avoid this happening with page modifications as well.
It's a pretty small change except for annoyingly making
log_heap_update() even more complicated. Note that this does not fix
tuple locking since that would be prohibitively expensive as described
in [2].
AI-generated repro and proposed fix attached.
- Melanie
[1] https://www.postgresql.org/message-id/tencent_2E870046716FD94285045E96505A2D4E2908%40qq.com
[2] https://www.postgresql.org/message-id/flat/tencent_2E870046716FD94285045E96505A2D4E2908%40qq.com#2b1...
Attachments:
repro-noop-vm-clear-portable.sh (application/x-shellscript, 9.3K) download
v1-0001-Register-VM-buffer-on-no-op-clears-so-redo-fixes-.patch (text/x-patch, 6.3K) download
From c5824d62c4d7cd24452f83a776fbfab8460a883a Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 16 Sep 2026 11:48:05 -0400
Subject: [PATCH v1] Register VM buffer on no-op clears so redo fixes a
divergent VM
Since ed62d26caca, heap WAL records clearing PD_ALL_VISIBLE (insert,
multi_insert, delete, update) only registered the VM buffer when clearing
its bits changed the VM page.
With CREATE DATABASE using strategy WAL_LOG and copying from a template, it's
possible to be in a scenario where the VM is clear on the primary and set on
the standby. Skipping clearing the VM during redo would then lead to data
corruption.
Though the reported CREATE DATABASE issue should be independently fixed,
harden the VM clear paths against VM divergence by always registering the VM
buffer when having to clear even PD_ALL_VISIBLE.
Tuple locking isn't fixed because it is pre-existing and hardening it in this
way would negatively impact performance significantly.
---
src/backend/access/heap/heapam.c | 59 ++++++++++++++++++++++++++------
1 file changed, 49 insertions(+), 10 deletions(-)
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 9ebb1b35d37..46815f52305 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -64,6 +64,7 @@ static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf,
Buffer vmbuffer_new, HeapTuple oldtup,
HeapTuple newtup, HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared,
+ bool vmbuffer_old_modified, bool vmbuffer_new_modified,
bool walLogical);
#ifdef USE_ASSERT_CHECKING
static void check_lock_if_inplace_updateable_rel(Relation relation,
@@ -2172,8 +2173,14 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
- if (vmbuffer_modified)
- XLogRegisterBuffer(HEAP_INSERT_BLKREF_VM, vmbuffer, 0);
+ /*
+ * Register the VM buffer even if its bits were already clear, so redo
+ * clears PD_ALL_VISIBLE's VM bits; the VM can be out-of-sync across a
+ * cluster.
+ */
+ if (clear_all_visible)
+ XLogRegisterBuffer(HEAP_INSERT_BLKREF_VM, vmbuffer,
+ vmbuffer_modified ? 0 : REGBUF_NO_CHANGE);
recptr = XLogInsert(RM_HEAP_ID, info);
@@ -2611,8 +2618,16 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
XLogRegisterData(xlrec, tupledata - scratch.data);
XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_HEAP, buffer,
REGBUF_STANDARD | bufflags);
- if (all_frozen_set || vmbuffer_modified)
- XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_VM, vmbuffer, 0);
+
+ /*
+ * Register the VM buffer when setting it all-frozen, or so redo
+ * clears PD_ALL_VISIBLE's VM bits even if they were already clear
+ * here (the VM can be out-of-sync across a cluster).
+ */
+ if (all_frozen_set || clear_all_visible)
+ XLogRegisterBuffer(HEAP_MULTI_INSERT_BLKREF_VM, vmbuffer,
+ (all_frozen_set || vmbuffer_modified) ?
+ 0 : REGBUF_NO_CHANGE);
XLogRegisterBufData(HEAP_MULTI_INSERT_BLKREF_HEAP, tupledata,
totaldatalen);
@@ -3142,8 +3157,14 @@ heap_delete(Relation relation, const ItemPointerData *tid,
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
- if (vmbuffer_modified)
- XLogRegisterBuffer(HEAP_DELETE_BLKREF_VM, vmbuffer, 0);
+ /*
+ * Register the VM buffer even if its bits were already clear, so redo
+ * clears PD_ALL_VISIBLE's VM bits; the VM can be out-of-sync across a
+ * cluster.
+ */
+ if (clear_all_visible)
+ XLogRegisterBuffer(HEAP_DELETE_BLKREF_VM, vmbuffer,
+ vmbuffer_modified ? 0 : REGBUF_NO_CHANGE);
recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE);
@@ -4274,14 +4295,25 @@ heap_update(Relation relation, const ItemPointerData *otid, HeapTuple newtup,
log_heap_new_cid(relation, heaptup);
}
+ /*
+ * When both heap pages are all-visible and share a VM page, that page
+ * is registered once as VM_NEW; pass the old slot as invalid to avoid
+ * registering the same buffer twice.
+ */
recptr = log_heap_update(relation, buffer,
- vmbuffer_modified ? vmbuffer : InvalidBuffer,
+ (clear_all_visible &&
+ !(clear_all_visible_new &&
+ vmbuffer == vmbuffer_new)) ?
+ vmbuffer : InvalidBuffer,
newbuf,
- vmbuffer_new_modified ? vmbuffer_new : InvalidBuffer,
+ clear_all_visible_new ?
+ vmbuffer_new : InvalidBuffer,
&oldtup, heaptup,
old_key_tuple,
clear_all_visible,
clear_all_visible_new,
+ vmbuffer_modified,
+ vmbuffer_new_modified,
walLogical);
if (newbuf != buffer)
{
@@ -9021,6 +9053,7 @@ log_heap_update(Relation reln, Buffer oldbuf, Buffer vmbuffer_old,
HeapTuple oldtup, HeapTuple newtup,
HeapTuple old_key_tuple,
bool all_visible_cleared, bool new_all_visible_cleared,
+ bool vmbuffer_old_modified, bool vmbuffer_new_modified,
bool walLogical)
{
xl_heap_update xlrec;
@@ -9233,15 +9266,21 @@ log_heap_update(Relation reln, Buffer oldbuf, Buffer vmbuffer_old,
* same VM page and both their VM bits were cleared, the caller passes
* only vmbuffer_new (mirroring the heap page convention where block 0 =
* new is always registered).
+ *
+ * A buffer is registered even if its bits were already clear, so redo
+ * clears PD_ALL_VISIBLE's VM bits (the VM can be out-of-sync across a
+ * cluster); use REGBUF_NO_CHANGE when the page was not modified.
*/
Assert((BufferIsInvalid(vmbuffer_old) && BufferIsInvalid(vmbuffer_new)) ||
(vmbuffer_old != vmbuffer_new));
if (BufferIsValid(vmbuffer_new))
- XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_NEW, vmbuffer_new, 0);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_NEW, vmbuffer_new,
+ vmbuffer_new_modified ? 0 : REGBUF_NO_CHANGE);
if (BufferIsValid(vmbuffer_old))
- XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_OLD, vmbuffer_old, 0);
+ XLogRegisterBuffer(HEAP_UPDATE_BLKREF_VM_OLD, vmbuffer_old,
+ vmbuffer_old_modified ? 0 : REGBUF_NO_CHANGE);
/* filtering by origin on a row level is much more efficient */
XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
--
2.43.0
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-09-17 09:24 Andrey Borodin <x4mmm@yandex-team.ru>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 1 reply; 57+ messages in thread
From: Andrey Borodin @ 2026-09-17 09:24 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
On 16 Sep 2026, Melanie Plageman wrote:
> Vacuum works very hard to never end up with
> PD_ALL_VISIBLE cleared and the VM set. Perhaps we should do everything
> we can to avoid this happening with page modifications as well.
Would it be practical to assert that the VM all-visible bit is clear
whenever we clear PD_ALL_VISIBLE? There may be reasons we don't already
do this that I'm missing, especially during recovery when pages can be
ahead of the record being replayed. It seems like a useful check if we
can find the right conditions for it.
Thank you!
Best regards, Andrey Borodin.
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-09-21 21:20 Melanie Plageman <melanieplageman@gmail.com>
parent: Andrey Borodin <x4mmm@yandex-team.ru>
0 siblings, 1 reply; 57+ messages in thread
From: Melanie Plageman @ 2026-09-21 21:20 UTC (permalink / raw)
To: Andrey Borodin <x4mmm@yandex-team.ru>; +Cc: Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
On Thu, Sep 17, 2026 at 5:25 AM Andrey Borodin <x4mmm@yandex-team.ru> wrote:
>
> On 16 Sep 2026, Melanie Plageman wrote:
> > Vacuum works very hard to never end up with
> > PD_ALL_VISIBLE cleared and the VM set. Perhaps we should do everything
> > we can to avoid this happening with page modifications as well.
>
> Would it be practical to assert that the VM all-visible bit is clear
> whenever we clear PD_ALL_VISIBLE? There may be reasons we don't already
> do this that I'm missing, especially during recovery when pages can be
> ahead of the record being replayed. It seems like a useful check if we
> can find the right conditions for it.
That wouldn't help in this case because it was already clear on the
primary. The problem is that the VM was set on the standby. I am going
to propose some patches to try and repair VM corruption in more cases
and warn when doing so so we know when things got out of sync. But we
can't catch this before it happens when clearing the VM on the primary
if it is already in the right state.
- Melanie
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-09-21 21:44 Melanie Plageman <melanieplageman@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 0 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-09-21 21:44 UTC (permalink / raw)
To: Andres Freund <andres@anarazel.de>; +Cc: Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
On Wed, Sep 16, 2026 at 12:10 PM Melanie Plageman
<melanieplageman@gmail.com> wrote:
>
> There is an interaction between CREATE DATABASE STRATEGY WAL_LOG and
> this fix that causes data corruption (identified while investigating
> [1]). The scenario is you have a heap page set PD_ALL_VISIBLE and a
> corresponding VM page set all-visible, you do a CREATE DATABASE
> STRATEGY WAL_LOG using that database as a template. The primary has
> the heap page set PDV (PD_ALL_VISIBLE) set and VM page AV set but the
> standby does NOT have the page set all-visible in the VM (it does have
> the heap page set PDV). After failover you have a primary with heap
> page PDV set and VM page not set. You modify a tuple on the page.
> Because the VM was already clear, we do not register the VM block in
> the WAL record. The standby replays this and clears the heap page
> PD_ALL_VISIBLE but leaves the VM set. This is data corruption leading
> to wrong results from index scans and worse if that standby is later
> promoted back to primary.
>
> We didn't have this problem before ed62d26caca because it cleared the
> VM during replay as long as PD_ALL_VISIBLE needed clearing. I mention
> in the thread [2] that I think we have to fix
> RelationCopyStorageUsingBuffer() to avoid producing this scenario
> because tuple locking already caused data corruption before
> ed62d26caca. However, for update/delete, etc, I'm wondering if we
> should harden this a bit and go back to registering a vm clear
> operation whenever PD_ALL_VISIBLE was set on the heap page. If this is
> a no-op on the primary for the VM, we register it with
> REBGUF_NO_CHANGE. Vacuum works very hard to never end up with
> PD_ALL_VISIBLE cleared and the VM set. Perhaps we should do everything
> we can to avoid this happening with page modifications as well.
We can't WAL-log all pages that might possibly be out-of-sync on the
standby even if we didn't need to modify them on the primary. And the
above approach would kind of set that precedent. As an alternative,
the attached patch makes sure the VM is clear on the standby even when
the VM block was not registered and emits a warning. It isn't
resilient to torn pages since we won't have emitted an FPI on the
primary, however, since it is making a best attempt to repair
corruption, I think that's fine.
This would only need to be backpatched to 19.
- Melanie
Attachments:
0001-Clear-divergent-visibility-map-bits-during-heap-redo.patch (text/x-patch, 8.3K) download
From 9f69805bb6491bb911da0a104ddc88187c04be7d Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Mon, 21 Sep 2026 16:01:22 -0400
Subject: [PATCH] Clear divergent visibility map bits during heap redo
When replaying a heap record that clears PD_ALL_VISIBLE but registered no
VM buffer (its clear was a no-op on the origin), clear the VM bit anyway.
The VM can diverge across a cluster, e.g. via CREATE DATABASE STRATEGY
WAL_LOG, and a standby must not keep a set VM bit over a page whose
PD_ALL_VISIBLE is clear. You can still get torn pages because you didn't
log an FPI on the primary, however it's a corruption repair scenario, so
that's fine. Only backpatch to 19 because <= 18 already have a fallback.
---
src/backend/access/heap/heapam_xlog.c | 89 +++++++++++++++++++++++++
src/backend/access/heap/visibilitymap.c | 33 +++++++++
src/include/access/visibilitymap.h | 3 +
3 files changed, 125 insertions(+)
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index 5fa1de09cfb..0b998fb5b5b 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -19,9 +19,56 @@
#include "access/visibilitymap.h"
#include "access/xlog.h"
#include "access/xlogutils.h"
+#include "common/relpath.h"
#include "storage/freespace.h"
#include "storage/standby.h"
+/*
+ * Clear visibility map bits for a heap block whose PD_ALL_VISIBLE flag the
+ * record being replayed cleared, but whose VM page the record did not
+ * register.
+ *
+ * The VM page is only registered if clearing the bits changed it, so the bits
+ * were already clear when the record was generated. They should be clear here
+ * too, since the VM is replicated through WAL. But the VM can be out of sync
+ * across a cluster (for instance, CREATE DATABASE STRATEGY WAL_LOG
+ * historically could produce out-of-sync VMs). It's incorrect for
+ * PD_ALL_VISIBLE to be clear and the VM to be set, so we must fix it.
+ *
+ * This is not fully resilient: the VM page is modified without a full-page
+ * image, so a torn write during a crash could leave it inconsistent until the
+ * page is next repaired. That is considered acceptable since this is an edge
+ * case in which we already have data corruption.
+ */
+static void
+heap_xlog_vm_clear_unregistered(RelFileLocator rlocator,
+ BlockNumber heap_blkno,
+ uint8 flags, XLogRecPtr lsn)
+{
+ Buffer vmbuffer = InvalidBuffer;
+
+ /*
+ * Read the VM page without extending the fork; a page that does not exist
+ * has no bits to clear. If the bits are already clear (the consistent
+ * case) there is nothing to do.
+ */
+ if (xlog_visibilitymap_get_status(rlocator, heap_blkno, &vmbuffer) & flags)
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ if (visibilitymap_clear(rlocator, heap_blkno, vmbuffer, flags))
+ {
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
+ ereport(WARNING,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("clearing out-of-sync visibility map bits for page %u of relation %s",
+ heap_blkno, relpathperm(rlocator, MAIN_FORKNUM).str)));
+ }
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else if (BufferIsValid(vmbuffer))
+ ReleaseBuffer(vmbuffer);
+}
+
/*
* Clear visibility map bits for a single heap block during heap redo.
*
@@ -46,7 +93,11 @@ heap_xlog_vm_clear(XLogReaderState *record,
Buffer vmbuffer = InvalidBuffer;
if (!XLogRecHasBlockRef(record, wal_vm_block_id))
+ {
+ heap_xlog_vm_clear_unregistered(target_locator, heap_blkno, flags,
+ lsn);
return;
+ }
/*
* If the vmbuffer was registered, use the recovery-specific routines to
@@ -764,6 +815,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
npage;
bool has_vm_old,
has_vm_new;
+ bool old_vm_cleared_via_new = false;
OffsetNumber offnum;
ItemId lp;
HeapTupleData oldtup;
@@ -842,8 +894,29 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
PageSetLSN(BufferGetPage(vmbuffer_new), lsn);
}
if (BufferIsValid(vmbuffer_new))
+ {
+ /*
+ * Remember whether oldblk's VM bits live on this page. If so,
+ * they are now up to date, whether we cleared them above or the
+ * page had already been replayed past this record.
+ */
+ old_vm_cleared_via_new =
+ (xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) &&
+ visibilitymap_pin_ok(oldblk, vmbuffer_new);
UnlockReleaseBuffer(vmbuffer_new);
+ }
}
+ else if (xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED)
+ {
+ /*
+ * If PD_ALL_VISIBLE was cleared on the new heap page and its
+ * corresponding VM page was not registered, ensure it is already
+ * clear or clear it.
+ */
+ heap_xlog_vm_clear_unregistered(rlocator, newblk,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ }
+
if (has_vm_old)
{
Buffer vmbuffer_old = InvalidBuffer;
@@ -860,6 +933,22 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
if (BufferIsValid(vmbuffer_old))
UnlockReleaseBuffer(vmbuffer_old);
}
+ else if ((xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED) &&
+ !old_vm_cleared_via_new)
+ {
+ /*
+ * VM_OLD is omitted when clearing the old page's VM bit was a no-op
+ * when the record was generated, because the bit was already clear.
+ * We get here in two such cases: either no VM_NEW was registered, or
+ * VM_NEW was registered but covers a different VM page than oldblk
+ * (the old and new heap pages map to different VM pages).
+ *
+ * If the VM has diverged on the standby, the bit may be set here, so
+ * clear it defensively.
+ */
+ heap_xlog_vm_clear_unregistered(rlocator, oldblk,
+ VISIBILITYMAP_VALID_BITS, lsn);
+ }
/*
* In normal operation, it is important to lock the two pages in
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index fe5ce437e1b..6136dbab4b0 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -12,6 +12,7 @@
*
* INTERFACE ROUTINES
* visibilitymap_clear - clear bits for one page in the visibility map
+ * xlog_visibilitymap_get_status - get status of bits during WAL replay
* visibilitymap_pin - pin a map page for setting a bit
* visibilitymap_pin_ok - check whether correct map page is already pinned
* visibilitymap_set - set bit(s) in a previously pinned page
@@ -189,6 +190,38 @@ visibilitymap_clear(RelFileLocator rlocator, BlockNumber heapBlk,
return cleared;
}
+/*
+ * Like visibilitymap_get_status(), but uses a RelFileLocator instead of a
+ * Relation, so it needs no relcache entry and can be used by redo routines.
+ * It never extends the fork: if the map page does not exist, there are no
+ * bits, so *vmbuf is set to InvalidBuffer and 0 is returned.
+ *
+ * On return *vmbuf holds the pinned (but unlocked) map page; the caller is
+ * responsible for releasing it. A caller that goes on to clear bits must lock
+ * it first.
+ */
+uint8
+xlog_visibilitymap_get_status(RelFileLocator rlocator, BlockNumber heapBlk,
+ Buffer *vmbuf)
+{
+ BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
+ uint32 mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
+ uint8 mapOffset = HEAPBLK_TO_OFFSET(heapBlk);
+ char *map;
+
+ Assert(InRecovery);
+
+ *vmbuf = XLogReadBufferExtended(rlocator, VISIBILITYMAP_FORKNUM, mapBlock,
+ RBM_NORMAL_NO_LOG, InvalidBuffer);
+ if (!BufferIsValid(*vmbuf))
+ return 0;
+
+ map = PageGetContents(BufferGetPage(*vmbuf));
+
+ /* A single byte read is atomic (see visibilitymap_get_status()). */
+ return ((map[mapByte] >> mapOffset) & VISIBILITYMAP_VALID_BITS);
+}
+
/*
* visibilitymap_pin - pin a map page for setting a bit
*
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index 165efd1c00e..8ed1e86778f 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -15,6 +15,7 @@
#define VISIBILITYMAP_H
#include "access/visibilitymapdefs.h"
+#include "access/xlogdefs.h"
#include "storage/block.h"
#include "storage/buf.h"
#include "storage/relfilelocator.h"
@@ -28,6 +29,8 @@
extern bool visibilitymap_clear(RelFileLocator rlocator, BlockNumber heapBlk,
Buffer vmbuf, uint8 flags);
+extern uint8 xlog_visibilitymap_get_status(RelFileLocator rlocator,
+ BlockNumber heapBlk, Buffer *vmbuf);
extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk,
Buffer *vmbuf);
extern bool visibilitymap_pin_ok(BlockNumber heapBlk, Buffer vmbuf);
--
2.43.0
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-09-22 06:54 Andrey Borodin <x4mmm@yandex-team.ru>
parent: Melanie Plageman <melanieplageman@gmail.com>
0 siblings, 1 reply; 57+ messages in thread
From: Andrey Borodin @ 2026-09-22 06:54 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
On 21 Sep 2026, Melanie Plageman wrote:
> That wouldn't help in this case because it was already clear on the
> primary.
Yes, my suggestion was too vague. I meant a check during redo that
would fail an assert-enabled buildfarm run and preserve enough state
to investigate. A WARNING can go unnoticed in a passing TAP test, as
David recently pointed out [0]. Could we make this fatal in assert
builds, once we have a condition that excludes pages legitimately
ahead of replay?
In heap_xlog_vm_clear_unregistered(), couldn't the VM page already have
a newer LSN? Consider a heap record R whose VM clear was a no-op,
followed by VACUUM setting that bit at S. If the VM page at S reaches
disk and crash recovery starts before R, the new helper would call this
corruption and move the page LSN backwards to R. Unlike
XLogReadBufferForRedo(), it does not check the page LSN. I did not
reproduce this, just seems possible.
Could we retain the original VM bits and page LSN in the diagnostic?
The WAL redo CONTEXT already identifies the record. A page-LSN check
also needs care: another heap block's bit can advance the same VM page's
LSN without repairing the bit we are interested in.
Also, if we accept a possible torn page from this repair, what happens
on the next recovery? The new reader uses RBM_NORMAL_NO_LOG, not the
VM's usual RBM_ZERO_ON_ERROR, so a checksum failure could stop recovery
before it gets to the repair.
For this repair, shouldn't we either use RBM_ZERO_ON_ERROR on every
relevant recovery read path, or require an FPI for the VM page since
the last checkpoint's redo point before allowing the modification?
There are quite a few moving parts in VM, and I don't yet have a clear
picture of how they all fit together. Sorry if some of these questions
are a distraction. I have a handful of open tickets about VM corruption,
and I hope we can track down and fix all possible issues.
Thank you!
Best regards, Andrey Borodin.
[0] https://www.postgresql.org/message-id/CAApHDvp4Oi4EuNixzR-mynNi4mKpVNsvbCDkNA_-yEtu-BeA3w%40mail.gma...
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-09-24 23:37 Melanie Plageman <melanieplageman@gmail.com>
parent: Andrey Borodin <x4mmm@yandex-team.ru>
0 siblings, 2 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-09-24 23:37 UTC (permalink / raw)
To: Andrey Borodin <x4mmm@yandex-team.ru>; +Cc: Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
On Tue, Sep 22, 2026 at 2:54 AM Andrey Borodin <x4mmm@yandex-team.ru> wrote:
>
> On 21 Sep 2026, Melanie Plageman wrote:
> > That wouldn't help in this case because it was already clear on the
> > primary.
>
> Yes, my suggestion was too vague. I meant a check during redo that
> would fail an assert-enabled buildfarm run and preserve enough state
> to investigate. A WARNING can go unnoticed in a passing TAP test, as
> David recently pointed out [0]. Could we make this fatal in assert
> builds, once we have a condition that excludes pages legitimately
> ahead of replay?
I don't think it makes sense to implement that here. There are no
other places in the code base where we error out in an assert build
and warn in a non-assert build. So, we'd be starting a new precedent.
And having it error out in non-assert builds for VM corruption will
probably make people mad.
I do think it makes sense to do something like what David is
suggesting in general, though.
One thing we could do here is make sure there is test coverage. I
committed a patch yesterday 5d84c76021c that changes verify_heapam()
to report corruption when PD_ALL_VISIBLE is clear and the VM is set.
We could add a test that uses pg_amcheck to detect this kind of
corruption. AI drafted one for me that used a lot of fancy perl that I
didn't yet evaluate, but I could look into it more.
While revisiting the warning I wrote in the patch you reviewed, I
realized I don't think it's the right thing to do. The warning as I
wrote it would warn whenever a WAL record is clearing the VM and the
primary's PD_ALL_VISIBLE was set and its visibility map was already
clear and the standby's visibility map is set. This doesn't seem right
because it will warn even if the standby doesn't actually have data
corruption. That is, if the standby has both PD_ALL_VISIBLE set and
visibility map set, it would still warn even though that is not
corruption. The warning is then basically about the primary being out
of sync with the standby. But there are many other combinations of the
primary and standby being out of sync (e.g. we try to set the
visibility map on the standby and it is already set). And, I don't
think we want to warn in all of these cases. Every combination of
PD_ALL_VISIBLE and VM set/clear differing between primary and standby
is a lot to warn on. And if you lost the whole VM on one node, for
example, you would get this warning for every page. And it's not
obvious why just this one case of divergence between primary and
standby is most important to warn on.
Instead, I suggest that, starting on master, we warn whenever there is
corruption on either node. On the primary it already prints a warning
when PD_ALL_VISIBLE is clear and the VM is set. Then it fixes it. We
should expand this on master and also wal-log it. We should also then
add a warning to the standby when it has PD_ALL_VISIBLE clear and the
VM set. Once we WAL-log fixing corruption on the primary, it should
only warn during recovery when the standby is corrupt and out of sync
with the primary. These changes are larger, so I am planning to only
propose it in a separate thread to master only. I would couple it with
hardening that makes it harder to get into these situations so the
warnings are less likely to be produced.
I plan to start a new thread with the patches proposed to master only
that adds a lot more corruption protection to the VM.
> In heap_xlog_vm_clear_unregistered(), couldn't the VM page already have
> a newer LSN? Consider a heap record R whose VM clear was a no-op,
> followed by VACUUM setting that bit at S. If the VM page at S reaches
> disk and crash recovery starts before R, the new helper would call this
> corruption and move the page LSN backwards to R. Unlike
> XLogReadBufferForRedo(), it does not check the page LSN. I did not
> reproduce this, just seems possible.
The ironic thing about this is that in the patch version you reviewed,
this behavior leaves things in a more correct state than prior to it.
However, I know it's wrong to decrease the LSN. Also it is wrong to
stamp a page with an LSN after making changes that weren't registered
in that WAL record. That's why we don't set the page LSN when clearing
the VM when the VM block is not registered in backbranches.
Just for fun, I'll describe why, in this particular case, my patch you
reviewed leaves things in a more correct state, though. VM page LSN
starts at 0, replay R and it does nothing to it. Replay S and it
advances the LSN from 0 -> 3. Then the VM page is persisted. Then we
crash. We replay R. It clears the VM and sets the LSN from 3 -> 2. We
replay S. It sets the VM and advances the LSN from 2 -> 3. On versions
18 and lower after the crash, R will clear the VM and not set the LSN.
S will see the LSN is already 3 and not do anything so the VM ends up
clear. That's not wrong, but it does create divergent state on primary
and standby. (19 without this patch wouldn't clear the VM at all,
which is wrong).
That being said, attached v2 is the correct approach for 19 and master
(for now). It does not stamp the LSN on the page when clearing the VM
when the VM block was not registered.
> Could we retain the original VM bits and page LSN in the diagnostic?
> The WAL redo CONTEXT already identifies the record.
I've removed the warning for now, but in the new more invasive patch
set I plan for master which will warn, I can include more details like
these. We can do things on master like have visibilitymap_clear()
return the original bits instead of just a boolean. That will make it
much easier to include details like this in the log message.
> A page-LSN check
> also needs care: another heap block's bit can advance the same VM page's
> LSN without repairing the bit we are interested in.
Yea, we don't want to gate the VM clear on the page LSN. This made me
think about how the VM can end up in an inconsistent state temporarily
during crash recovery. So, when I do add the warning to master, I'll
make sure it is only when we've reached a consistent state (I think
you mention that in your earlier comment).
> Also, if we accept a possible torn page from this repair, what happens
> on the next recovery? The new reader uses RBM_NORMAL_NO_LOG, not the
> VM's usual RBM_ZERO_ON_ERROR, so a checksum failure could stop recovery
> before it gets to the repair.
Yea, I did this on purpose because since ed62d26caca, I didn't read
the VM with ZERO_ON_ERROR, which I thought was okay since we were
always registering the VM blocks in the wal record for both setting
and clearing.
However, this isn't a good idea when 17 and 18 have a fallback path
that lets us read the VM in recovery (when clearing it) in a way that
could result in a torn page. Then we've created a way for the VM to
get corrupt and made it error out reading it during recovery.
> For this repair, shouldn't we either use RBM_ZERO_ON_ERROR on every
> relevant recovery read path, or require an FPI for the VM page since
> the last checkpoint's redo point before allowing the modification?
Yea, I've thought about this a lot the last few days. On back branches
(and as a stop-gap in master), I think we have to use
RBM_ZERO_ON_ERROR everywhere and be okay with tearing pages when
reading VM pages during recovery.
On master, going forward, we should register the VM whenever
PD_ALL_VISIBLE is being cleared. And then we should not use
RBM_ZERO_ON_ERROR for setting or clearing. For setting, that means
adding a new mode that will extend the VM if the page doesn't exist
but error out if the page is corrupt (RBM_ZERO_ON_MISSING). Then we
can make the zero_damaged_pages GUC per fork. I have four or five
ideas for making master more robust to VM corruption. And there are
still some open questions, like what to do about tuple locking, but
that can be discussed in more depth there.
But I think as long as we have a code path that we know leads to torn
pages, we have to zero those pages on error when reading them.
> There are quite a few moving parts in VM, and I don't yet have a clear
> picture of how they all fit together. Sorry if some of these questions
> are a distraction. I have a handful of open tickets about VM corruption,
> and I hope we can track down and fix all possible issues.
I certainly think we can harden master by making some bigger changes.
I have a draft of the changes and hope to post by mid next week.
For now, I've attached four patches that are much narrower fixes and
are backpatchable:
v2-0001 is 19-only and fixes the divergent VM bits on the standby by
reading the VM when the VM block wasn't registered
master-v2-0001 is the stop-gap for master that does the same thing but
in a different way because it doesn't have a fake relcache entry
v2-0002 fixes VM clear to read the VM with RBM_ZERO_ON_ERROR. This
should apply master -> 17
v2-0003 is a fix for 19 and master for an issue I found while working
on this where the VM wasn't guaranteed to be dirty by the time we
logged setting it in heap_page_prune_and_freeze(). This can happen if
it was already set all-frozen and all-visible but it contained any
non-frozen tuples (i.e. VM is corrupt with incorrect all-frozen bit).
The VM set operation would be a no-op and no one would fix the corrupt
all-frozen bit. This corruption was never fixed, but in 19 the
vm-setting code was rearranged to expect that if we try and set the
VM, we will dirty it. For that to be true, we have to fix this
corruption. And, since the VM corruption detection function is new in
19, it is easiest to just backpatch it that far.
- Melanie
Attachments: download all (4 files, .tar.gz)
v2-0001-Clear-divergent-visibility-map-bits-during-heap-r.patch (text/x-patch, 3.4K) download
From 3bd4475e117a30219b28239709741ac60bf7aec2 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 23 Sep 2026 10:53:09 -0400
Subject: [PATCH v2 1/3] Clear divergent visibility map bits during heap redo
When replaying a heap record that clears PD_ALL_VISIBLE but did not
register a VM buffer because the VM was already clear on the primary,
clear the VM bit anyway.
The VM can diverge across a cluster, e.g. via CREATE DATABASE STRATEGY
WAL_LOG, and a standby must not keep a set VM bit over a page whose
PD_ALL_VISIBLE is clear. You can still get torn pages because you didn't
log an FPI on the primary, however the VM is usually read with
RBM_ZERO_ON_ERROR, so it is considered sufficient. Only backpatch to 19
because <= 18 already have a fallback.
Backpatch-through: 19
---
src/backend/access/heap/heapam_xlog.c | 49 ++++++++++++++++++++++-----
1 file changed, 40 insertions(+), 9 deletions(-)
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index fae3b477c09..c65804a6256 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -22,6 +22,33 @@
#include "storage/freespace.h"
#include "storage/standby.h"
+/*
+ * Clear visibility map bits for a heap block when the WAL record clearing it
+ * did not register the VM block. This handles cases where the VM is
+ * out-of-sync between the primary and standby (for instance, CREATE DATABASE
+ * STRATEGY WAL_LOG historically could cause this).
+ *
+ * This is not fully resilient: the VM page is modified without a full-page
+ * image, so a torn write during a crash could leave it inconsistent until the
+ * page is next repaired. That is considered acceptable since the VM is zeroed
+ * on error when reading it.
+ */
+static void
+heap_xlog_vm_clear_unregistered(Relation reln, BlockNumber heap_blkno,
+ uint8 flags)
+{
+ Buffer vmbuffer = InvalidBuffer;
+
+ if (visibilitymap_get_status(reln, heap_blkno, &vmbuffer) & flags)
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear(reln, heap_blkno, vmbuffer, flags);
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else if (BufferIsValid(vmbuffer))
+ ReleaseBuffer(vmbuffer);
+}
+
/*
* Clear visibility map bits for a single heap block during heap redo.
*
@@ -46,22 +73,26 @@ heap_xlog_vm_clear(XLogReaderState *record,
Relation reln = CreateFakeRelcacheEntry(target_locator);
Buffer vmbuffer = InvalidBuffer;
+ if (!XLogRecHasBlockRef(record, wal_vm_block_id))
+ {
+ heap_xlog_vm_clear_unregistered(reln, heap_blkno, flags);
+ FreeFakeRelcacheEntry(reln);
+ return;
+ }
+
/*
* If the vmbuffer was registered, use the recovery-specific routines to
* read it. These will either apply an FPI or indicate that we should
* clear the requested bits ourselves.
*/
- if (XLogRecHasBlockRef(record, wal_vm_block_id))
+ if (XLogReadBufferForRedo(record, wal_vm_block_id,
+ &vmbuffer) == BLK_NEEDS_REDO)
{
- if (XLogReadBufferForRedo(record, wal_vm_block_id,
- &vmbuffer) == BLK_NEEDS_REDO)
- {
- if (visibilitymap_clear(reln, heap_blkno, vmbuffer, flags))
- PageSetLSN(BufferGetPage(vmbuffer), lsn);
- }
- if (BufferIsValid(vmbuffer))
- UnlockReleaseBuffer(vmbuffer);
+ if (visibilitymap_clear(reln, heap_blkno, vmbuffer, flags))
+ PageSetLSN(BufferGetPage(vmbuffer), lsn);
}
+ if (BufferIsValid(vmbuffer))
+ UnlockReleaseBuffer(vmbuffer);
FreeFakeRelcacheEntry(reln);
}
--
2.43.0
master-v2-0001-Clear-divergent-visibility-map-bits-during-.patch (text/x-patch, 5.4K) download
From db07f4d74e0632ddd39ea80fa73a719aa8d9a659 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 23 Sep 2026 16:58:54 -0400
Subject: [PATCH vmaster2 1/3] Clear divergent visibility map bits during heap
redo
When replaying a heap record that clears PD_ALL_VISIBLE but did not
register a VM buffer because the VM was already clear on the primary,
clear the VM bit anyway.
The VM can diverge across a cluster, e.g. via CREATE DATABASE STRATEGY
WAL_LOG, and a standby must not keep a set VM bit over a page whose
PD_ALL_VISIBLE is clear. You can still get torn pages because you didn't
log an FPI on the primary, however the VM is usually read with
RBM_ZERO_ON_ERROR, so it is considered sufficient. Only backpatch to 19
because <= 18 already have a fallback.
The fix differs on master because it no longer makes a fake relcache
entry in recovery, so it needed a dedicated function to read the
unregistered VM page. This is a stopgap so that the issue is fixed
across branches. In the future, master should implement a more robust
fix.
Backpatch-through: 19
---
src/backend/access/heap/heapam_xlog.c | 30 ++++++++++++++++++++++++
src/backend/access/heap/visibilitymap.c | 31 +++++++++++++++++++++++++
src/include/access/visibilitymap.h | 2 ++
3 files changed, 63 insertions(+)
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index 5fa1de09cfb..e1c68be698d 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -22,6 +22,33 @@
#include "storage/freespace.h"
#include "storage/standby.h"
+/*
+ * Clear visibility map bits for a heap block when the WAL record clearing it
+ * did not register the VM block. This handles cases where the VM is
+ * out-of-sync between the primary and standby (for instance, CREATE DATABASE
+ * STRATEGY WAL_LOG historically could cause this).
+ *
+ * This is not fully resilient: the VM page is modified without a full-page
+ * image, so a torn write during a crash could leave it inconsistent until the
+ * page is next repaired. That is considered acceptable since the VM is zeroed
+ * on error when reading it.
+ */
+static void
+heap_xlog_vm_clear_unregistered(RelFileLocator rlocator, BlockNumber heap_blkno,
+ uint8 flags)
+{
+ Buffer vmbuffer = InvalidBuffer;
+
+ if (xlog_visibilitymap_get_status(rlocator, heap_blkno, &vmbuffer) & flags)
+ {
+ LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE);
+ visibilitymap_clear(rlocator, heap_blkno, vmbuffer, flags);
+ UnlockReleaseBuffer(vmbuffer);
+ }
+ else if (BufferIsValid(vmbuffer))
+ ReleaseBuffer(vmbuffer);
+}
+
/*
* Clear visibility map bits for a single heap block during heap redo.
*
@@ -46,7 +73,10 @@ heap_xlog_vm_clear(XLogReaderState *record,
Buffer vmbuffer = InvalidBuffer;
if (!XLogRecHasBlockRef(record, wal_vm_block_id))
+ {
+ heap_xlog_vm_clear_unregistered(target_locator, heap_blkno, flags);
return;
+ }
/*
* If the vmbuffer was registered, use the recovery-specific routines to
diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c
index fe5ce437e1b..d7ba6b8ca20 100644
--- a/src/backend/access/heap/visibilitymap.c
+++ b/src/backend/access/heap/visibilitymap.c
@@ -12,6 +12,7 @@
*
* INTERFACE ROUTINES
* visibilitymap_clear - clear bits for one page in the visibility map
+ * xlog_visibilitymap_get_status - get status of bits during WAL replay
* visibilitymap_pin - pin a map page for setting a bit
* visibilitymap_pin_ok - check whether correct map page is already pinned
* visibilitymap_set - set bit(s) in a previously pinned page
@@ -189,6 +190,36 @@ visibilitymap_clear(RelFileLocator rlocator, BlockNumber heapBlk,
return cleared;
}
+/*
+ * Like visibilitymap_get_status(), but uses a RelFileLocator instead of a
+ * Relation, so it needs no relcache entry and can be used by redo routines.
+ *
+ * On return *vmbuf holds the pinned (but unlocked) map page; the caller is
+ * responsible for releasing it. A caller that goes on to clear bits must lock
+ * it first.
+ */
+uint8
+xlog_visibilitymap_get_status(RelFileLocator rlocator, BlockNumber heapBlk,
+ Buffer *vmbuf)
+{
+ BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
+ uint32 mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
+ uint8 mapOffset = HEAPBLK_TO_OFFSET(heapBlk);
+ char *map;
+
+ Assert(InRecovery);
+
+ *vmbuf = XLogReadBufferExtended(rlocator, VISIBILITYMAP_FORKNUM, mapBlock,
+ RBM_ZERO_ON_ERROR, InvalidBuffer);
+ if (!BufferIsValid(*vmbuf))
+ return 0;
+
+ map = PageGetContents(BufferGetPage(*vmbuf));
+
+ /* A single byte read is atomic (see visibilitymap_get_status()). */
+ return ((map[mapByte] >> mapOffset) & VISIBILITYMAP_VALID_BITS);
+}
+
/*
* visibilitymap_pin - pin a map page for setting a bit
*
diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h
index 165efd1c00e..47b28f311f4 100644
--- a/src/include/access/visibilitymap.h
+++ b/src/include/access/visibilitymap.h
@@ -28,6 +28,8 @@
extern bool visibilitymap_clear(RelFileLocator rlocator, BlockNumber heapBlk,
Buffer vmbuf, uint8 flags);
+extern uint8 xlog_visibilitymap_get_status(RelFileLocator rlocator,
+ BlockNumber heapBlk, Buffer *vmbuf);
extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk,
Buffer *vmbuf);
extern bool visibilitymap_pin_ok(BlockNumber heapBlk, Buffer vmbuf);
--
2.43.0
v2-0002-Read-visibility-map-pages-with-RBM_ZERO_ON_ERROR-.patch (text/x-patch, 2.9K) download
From 66e86a8bedaff7c9d47f05b7e025c12f8a9747ac Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 23 Sep 2026 11:59:53 -0400
Subject: [PATCH v2 2/3] Read visibility map pages with RBM_ZERO_ON_ERROR in VM
clear redo
ed62d26caca started registering VM blocks when clearing the VM which is
required for protection against torn pages as well as for correct
incremental backups. However, it read the VM pages in recovery with
RBM_NORMAL which errors out when it encounters a corrupt page. This is
usually desirable, however, we still retain code paths that modify the
VM in recovery without the block having been registered. A crash while
modifying the VM page could lead to a corrupt page and no FPI to recover
it. As long as we can trivially produce corrupt pages during recovery
through our own redo mechanism, we shouldn't error out when reading a
corrupt VM page.
Make clearing the VM read the page with RBM_ZERO_ON_ERROR. This is
consistent with the VM's other redo paths which set the VM bit
(heap_xlog_prune_freeze() and heap_xlog_multi_insert()).
Backpatch-through: 17
---
src/backend/access/heap/heapam_xlog.c | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c
index c65804a6256..4d99d99080b 100644
--- a/src/backend/access/heap/heapam_xlog.c
+++ b/src/backend/access/heap/heapam_xlog.c
@@ -85,8 +85,9 @@ heap_xlog_vm_clear(XLogReaderState *record,
* read it. These will either apply an FPI or indicate that we should
* clear the requested bits ourselves.
*/
- if (XLogReadBufferForRedo(record, wal_vm_block_id,
- &vmbuffer) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedoExtended(record, wal_vm_block_id,
+ RBM_ZERO_ON_ERROR, false,
+ &vmbuffer) == BLK_NEEDS_REDO)
{
if (visibilitymap_clear(reln, heap_blkno, vmbuffer, flags))
PageSetLSN(BufferGetPage(vmbuffer), lsn);
@@ -850,8 +851,9 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Assert(xlrec->flags & XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED);
- if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_NEW,
- &vmbuffer_new) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedoExtended(record, HEAP_UPDATE_BLKREF_VM_NEW,
+ RBM_ZERO_ON_ERROR, false,
+ &vmbuffer_new) == BLK_NEEDS_REDO)
{
/*
* If both the old and new heap pages were all-visible and their
@@ -888,8 +890,9 @@ heap_xlog_update(XLogReaderState *record, bool hot_update)
Assert(xlrec->flags & XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED);
- if (XLogReadBufferForRedo(record, HEAP_UPDATE_BLKREF_VM_OLD,
- &vmbuffer_old) == BLK_NEEDS_REDO)
+ if (XLogReadBufferForRedoExtended(record, HEAP_UPDATE_BLKREF_VM_OLD,
+ RBM_ZERO_ON_ERROR, false,
+ &vmbuffer_old) == BLK_NEEDS_REDO)
{
if (visibilitymap_clear(reln, oldblk, vmbuffer_old,
VISIBILITYMAP_VALID_BITS))
--
2.43.0
v2-0003-Detect-and-repair-a-stale-all-frozen-visibility-m.patch (text/x-patch, 3.1K) download
From 0bda92f7fe32f24b7b9b0b9caaac111c79c3b434 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Thu, 24 Sep 2026 16:42:36 -0400
Subject: [PATCH v2 3/3] Detect and repair a stale all-frozen visibility map
bit
visibilitymap_set() does not clear any bits, so if vacuum finds that
every tuple on the page is visible but not every tuple is frozen and yet
the page is marked all-frozen in the VM, it wouldn't repair it.
Add a VM_CORRUPT_STALE_ALL_FROZEN case to heap_page_fix_vm_corruption()
that will clear the VM bits. The normal path will then set the
all-visible bit separately. After this change, we can be sure that
visibilitymap_set() will modify the VM buffer and mark it dirty.
Backpatch-through: 19
---
src/backend/access/heap/pruneheap.c | 33 +++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)
diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c
index 833f6f0395c..2ddcbecff3a 100644
--- a/src/backend/access/heap/pruneheap.c
+++ b/src/backend/access/heap/pruneheap.c
@@ -196,6 +196,8 @@ typedef enum VMCorruptionType
VM_CORRUPT_LPDEAD,
/* Tuple not visible to all transactions on a page marked all-visible */
VM_CORRUPT_TUPLE_VISIBILITY,
+ /* Page marked all-frozen in the VM but not actually all-frozen */
+ VM_CORRUPT_STALE_ALL_FROZEN,
} VMCorruptionType;
/* Local functions */
@@ -944,6 +946,23 @@ heap_page_fix_vm_corruption(PruneState *prstate, OffsetNumber offnum,
relname, prstate->block)));
do_clear_vm = true;
break;
+
+ case VM_CORRUPT_STALE_ALL_FROZEN:
+
+ /*
+ * We examined every tuple on the page and found that the page is
+ * all-visible but not all-frozen, yet the VM marks it all-frozen.
+ * The page-level PD_ALL_VISIBLE flag is still correct, so only
+ * the VM is wrong. Clear both the VM bits. All-visible will be
+ * set again through the normal path.
+ */
+ ereport(WARNING,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg("page marked all-frozen in the visibility map is not all-frozen"),
+ errcontext("relation \"%s\", page %u",
+ relname, prstate->block)));
+ do_clear_vm = true;
+ break;
}
Assert(do_clear_heap || do_clear_vm);
@@ -1259,6 +1278,20 @@ heap_page_prune_and_freeze(PruneFreezeParams *params,
Assert(!prstate.set_all_visible || prstate.attempt_set_vm);
Assert(!prstate.set_all_visible || (prstate.lpdead_items == 0));
+ /*
+ * If we examined every tuple on the page and found that the page is
+ * all-visible but not all-frozen, yet the VM marks it all-frozen, that
+ * all-frozen bit is corrupt. Repair the VM. We will set it back to
+ * all-visible later along with the other changes. Note that this must be
+ * done after set_all_visible and set_all_frozen are finalized above to
+ * account for dead items and unfrozen tuples.
+ */
+ if (prstate.attempt_freeze && prstate.set_all_visible &&
+ !prstate.set_all_frozen &&
+ (prstate.old_vmbits & VISIBILITYMAP_ALL_FROZEN))
+ heap_page_fix_vm_corruption(&prstate, InvalidOffsetNumber,
+ VM_CORRUPT_STALE_ALL_FROZEN);
+
do_set_vm = heap_page_will_set_vm(&prstate, params->reason, do_prune, do_freeze);
/*
--
2.43.0
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-10-06 06:38 Andrey Borodin <x4mmm@yandex-team.ru>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 0 replies; 57+ messages in thread
From: Andrey Borodin @ 2026-10-06 06:38 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
Hi Melanie,
On 24 Sep 2026, Melanie Plageman wrote:
> v2-0001 is 19-only and fixes the divergent VM bits on the standby
The UPDATE fallback from v1 is missing in both versions of 0001.
heap_xlog_update() handles VM_OLD/VM_NEW itself, without calling
heap_xlog_vm_clear(), so an unregistered VM page is still skipped.
I reproduced this with UPDATE in your CREATE DATABASE/failover scenario
on master. Could we add the fallback there too?
Separately, Shihao's v3 [0] adds PageInit() fix on top of your 0002.
Thank you!
Best regards, Andrey Borodin.
[0] https://www.postgresql.org/message-id/CAGRkXqSZXoazu1x48pz2SuW_xHYQDdRcbp2HNCfRNm_WLF8aYA%40mail.gma...
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-10-13 19:58 Melanie Plageman <melanieplageman@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 0 replies; 57+ messages in thread
From: Melanie Plageman @ 2026-10-13 19:58 UTC (permalink / raw)
To: Andrey Borodin <x4mmm@yandex-team.ru>; +Cc: Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Robert Haas <robertmhaas@gmail.com>; Heikki Linnakangas <hlinnaka@iki.fi>
On Thu, Sep 24, 2026 at 7:37 PM Melanie Plageman
<melanieplageman@gmail.com> wrote:
>
> On Tue, Sep 22, 2026 at 2:54 AM Andrey Borodin <x4mmm@yandex-team.ru> wrote:
>
> > Also, if we accept a possible torn page from this repair, what happens
> > on the next recovery? The new reader uses RBM_NORMAL_NO_LOG, not the
> > VM's usual RBM_ZERO_ON_ERROR, so a checksum failure could stop recovery
> > before it gets to the repair.
>
> Yea, I did this on purpose because since ed62d26caca, I didn't read
> the VM with ZERO_ON_ERROR, which I thought was okay since we were
> always registering the VM blocks in the wal record for both setting
> and clearing.
>
> However, this isn't a good idea when 17 and 18 have a fallback path
> that lets us read the VM in recovery (when clearing it) in a way that
> could result in a torn page. Then we've created a way for the VM to
> get corrupt and made it error out reading it during recovery.
>
> > For this repair, shouldn't we either use RBM_ZERO_ON_ERROR on every
> > relevant recovery read path, or require an FPI for the VM page since
> > the last checkpoint's redo point before allowing the modification?
>
> Yea, I've thought about this a lot the last few days. On back branches
> (and as a stop-gap in master), I think we have to use
> RBM_ZERO_ON_ERROR everywhere and be okay with tearing pages when
> reading VM pages during recovery.
I've thought about this more, and I've changed my mind. I see no way
that using RBM_NORMAL in the way that I do in heap_xlog_vm_clear() can
lead to halting recovery from a VM page that we corrupted ourselves
(of course disk corruption can always happen). The exception is that
other VM truncation thread [1], but that can be fixed with
XLogTruncateRelation().
If you can come up with a repro, perhaps I'm wrong, but, even in the
case of divergence between primary and standby, the normal
visibilitymap_clear() fallback path does RBM_ZERO_ON_ERROR. So, even
if you corrupted a VM page by crashing on the standby, any record
replayed with XLogReadBufferForRedo() is guaranteed to have an FPI if
it needs one.
heap_xlog_vm_clear only reaches RBM_NORMAL on the registered branch,
i.e. when the primary actually cleared a set VM bit. That record is
FPI-free only if the VM page was already modified since the primary's
redo point — meaning an earlier FPI-bearing record for that same page
exists in the stream.
For redo to read a torn page, it must resume past that earlier record,
which needs a restartpoint between them. But restartpoints only ever
derive from a replayed primary checkpoint record. A checkpoint there
moves the redo point, which makes the clear first-since-redo-point
again — so it carries its own FPI.
Either the redo resumes before the protecting record (which re-images
the page) or the clear itself gets an FPI. The RBM_NORMAL read never
sees an unhealed torn page, so it can't halt recovery.
> v2-0001 is 19-only and fixes the divergent VM bits on the standby by
> reading the VM when the VM block wasn't registered
> master-v2-0001 is the stop-gap for master that does the same thing but
> in a different way because it doesn't have a fake relcache entry
These should still be applied. I plan to fix the CREATE DATABASE USING
STRATEGY WAL_LOG issue as well. But, if divergence can happen another
way, we'd still need this failsafe clear-the-vm patches on 19 and
master. It doesn't help with tuple locking clearing all-frozen though.
> v2-0002 fixes VM clear to read the VM with RBM_ZERO_ON_ERROR. This
> should apply master -> 17
I no longer think we should do this.
> v2-0003 is a fix for 19 and master for an issue I found while working
> on this where the VM wasn't guaranteed to be dirty by the time we
> logged setting it in heap_page_prune_and_freeze(). This can happen if
> it was already set all-frozen and all-visible but it contained any
> non-frozen tuples (i.e. VM is corrupt with incorrect all-frozen bit).
> The VM set operation would be a no-op and no one would fix the corrupt
> all-frozen bit. This corruption was never fixed, but in 19 the
> vm-setting code was rearranged to expect that if we try and set the
> VM, we will dirty it. For that to be true, we have to fix this
> corruption. And, since the VM corruption detection function is new in
> 19, it is easiest to just backpatch it that far.
This is still worth doing.
- Melanie
[1] https://www.postgresql.org/message-id/CAGRkXqSZXoazu1x48pz2SuW_xHYQDdRcbp2HNCfRNm_WLF8aYA%40mail.gma...
^ permalink raw reply [nested|flat] 57+ messages in thread
* Re: Why clearing the VM doesn't require registering vm buffer in wal record
@ 2026-10-26 10:30 Amit Kapila <amit.kapila16@gmail.com>
parent: Melanie Plageman <melanieplageman@gmail.com>
1 sibling, 0 replies; 57+ messages in thread
From: Amit Kapila @ 2026-10-26 10:30 UTC (permalink / raw)
To: Melanie Plageman <melanieplageman@gmail.com>; +Cc: Fujii Masao <masao.fujii@gmail.com>; Robert Haas <robertmhaas@gmail.com>; Andrey Borodin <x4mmm@yandex-team.ru>; Andres Freund <andres@anarazel.de>; Matthias van de Meent <boekewurm+postgres@gmail.com>; PostgreSQL Hackers <pgsql-hackers@lists.postgresql.org>; Heikki Linnakangas <hlinnaka@iki.fi>
On Sat, Jul 18, 2026 at 2:22 AM Melanie Plageman
<melanieplageman@gmail.com> wrote:
>
> I've committed this.
>
While reading the code committed as ed62d26cac, I wonder whether the
change in commit requires an update in the comments atop
visibilitymap.c. See following observations:
1. The following text on WAL-logging doesn't seem to match the changed
code. "Changes to the visibility map bits are not separately
WAL-logged. Callers must make sure that whenever a visibility map bit
is cleared, the bit is cleared on WAL replay of the updating
operation…"
The VM page is still not logged by its own record. But after commit
ed62d26cac, a caller that clears a bit must register the VM buffer
(block ref *_BLKREF_VM) in the heap record. It must also PageSetLSN()
the VM page. This makes me feel that the above quoted text could be
updated.
2. The LOCKING section is now wrong about the lock order and critical section.
It describes only a pin on the VM page before locking the heap buffer,
and the pin/relock race. Callers now also take an exclusive lock on
the VM buffer before START_CRIT_SECTION() and hold it until after
XLogInsert() and release it before the heap buffer lock.
3. The last LOCKING paragraph also needs an update. I am referring to:
"But when a bit is cleared, we don't have to do that [update the VM
page LSN] because it's always safe to clear a bit in the map from
correctness point of view."
The commit ed62d26cac now sets the VM page LSN on clear when
vmbuffer_modified is true.
--
With Regards,
Amit Kapila.
^ permalink raw reply [nested|flat] 57+ messages in thread
end of thread, other threads:[~2026-10-26 10:30 UTC | newest]
Thread overview: 57+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2025-12-09 19:17 [PATCH v1 5/6] Handle pg_get_expr default args in system_functions.sql Mark Wong <markwkm@gmail.com>
2025-12-09 19:17 [PATCH v4 5/6] Handle pg_get_expr default args in system_functions.sql Mark Wong <markwkm@gmail.com>
2025-12-09 19:17 [PATCH v7 5/6] Handle pg_get_expr default args in system_functions.sql Mark Wong <markwkm@gmail.com>
2025-12-09 19:17 [PATCH v8 5/6] Handle pg_get_expr default args in system_functions.sql Mark Wong <markwkm@gmail.com>
2025-12-09 19:17 [PATCH v5 5/6] Handle pg_get_expr default args in system_functions.sql Mark Wong <markwkm@gmail.com>
2026-03-05 19:56 Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-03-05 20:16 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andres Freund <andres@anarazel.de>
2026-03-05 20:23 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Matthias van de Meent <boekewurm+postgres@gmail.com>
2026-03-05 20:38 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andres Freund <andres@anarazel.de>
2026-03-05 21:01 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andres Freund <andres@anarazel.de>
2026-03-12 11:25 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Yura Sokolov <y.sokolov@postgrespro.ru>
2026-03-12 12:22 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Yura Sokolov <y.sokolov@postgrespro.ru>
2026-04-30 21:44 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-05-01 16:41 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Robert Haas <robertmhaas@gmail.com>
2026-05-01 16:58 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andres Freund <andres@anarazel.de>
2026-05-01 17:30 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Robert Haas <robertmhaas@gmail.com>
2026-05-01 19:14 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andres Freund <andres@anarazel.de>
2026-05-06 21:46 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-05-07 12:54 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Tomas Vondra <tomas@vondra.me>
2026-06-24 00:01 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-06-23 23:55 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-06-26 20:05 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Robert Haas <robertmhaas@gmail.com>
2026-06-27 10:41 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andrey Borodin <x4mmm@yandex-team.ru>
2026-06-29 07:46 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andrey Borodin <x4mmm@yandex-team.ru>
2026-07-10 19:12 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-06-29 14:42 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Robert Haas <robertmhaas@gmail.com>
2026-07-10 19:06 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-07-14 19:42 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-07-15 21:58 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-07-16 01:30 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Fujii Masao <masao.fujii@gmail.com>
2026-07-16 22:35 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-07-17 20:51 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-07-20 02:34 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Tom Lane <tgl@sss.pgh.pa.us>
2026-07-20 14:56 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-08-04 22:13 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-10-26 10:30 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Amit Kapila <amit.kapila16@gmail.com>
2026-07-17 21:18 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andres Freund <andres@anarazel.de>
2026-07-15 18:27 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Robert Haas <robertmhaas@gmail.com>
2026-07-15 19:34 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-07-15 20:10 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Robert Haas <robertmhaas@gmail.com>
2026-09-16 16:10 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-09-17 09:24 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andrey Borodin <x4mmm@yandex-team.ru>
2026-09-21 21:20 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-09-22 06:54 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andrey Borodin <x4mmm@yandex-team.ru>
2026-09-24 23:37 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-10-06 06:38 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andrey Borodin <x4mmm@yandex-team.ru>
2026-10-13 19:58 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-09-21 21:44 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Melanie Plageman <melanieplageman@gmail.com>
2026-03-05 23:11 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Robert Haas <robertmhaas@gmail.com>
2026-03-06 00:43 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andres Freund <andres@anarazel.de>
2026-03-06 16:49 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Robert Haas <robertmhaas@gmail.com>
2026-03-12 17:05 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andres Freund <andres@anarazel.de>
2026-03-12 19:27 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Tomas Vondra <tomas@vondra.me>
2026-03-12 21:26 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andres Freund <andres@anarazel.de>
2026-03-12 21:54 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Tomas Vondra <tomas@vondra.me>
2026-03-12 23:34 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Andres Freund <andres@anarazel.de>
2026-03-17 16:06 ` Re: Why clearing the VM doesn't require registering vm buffer in wal record Robert Haas <robertmhaas@gmail.com>
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox