public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v8 4/8] Propagate changes to indisclustered to child/parents
33+ messages / 5 participants
[nested] [flat]
* [PATCH v8 4/8] Propagate changes to indisclustered to child/parents
@ 2020-10-07 03:11 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Justin Pryzby @ 2020-10-07 03:11 UTC (permalink / raw)
---
src/backend/commands/cluster.c | 109 ++++++++++++++++----------
src/backend/commands/indexcmds.c | 2 +
src/test/regress/expected/cluster.out | 46 +++++++++++
src/test/regress/sql/cluster.sql | 11 +++
4 files changed, 125 insertions(+), 43 deletions(-)
diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c
index cb4fc350c6..5c08f0642e 100644
--- a/src/backend/commands/cluster.c
+++ b/src/backend/commands/cluster.c
@@ -74,6 +74,7 @@ static void rebuild_relation(Relation OldHeap, Oid indexOid, bool verbose);
static void copy_table_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex,
bool verbose, bool *pSwapToastByContent,
TransactionId *pFreezeXid, MultiXactId *pCutoffMulti);
+static void set_indisclustered(Oid indexOid, bool isclustered, Relation pg_index);
static List *get_tables_to_cluster(MemoryContext cluster_context);
static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context,
Oid indexOid);
@@ -516,66 +517,88 @@ check_index_is_clusterable(Relation OldHeap, Oid indexOid, bool recheck, LOCKMOD
index_close(OldIndex, NoLock);
}
+/*
+ * Helper for mark_index_clustered
+ * Mark a single index as clustered or not.
+ * pg_index is passed by caller to avoid repeatedly re-opening it.
+ */
+static void
+set_indisclustered(Oid indexOid, bool isclustered, Relation pg_index)
+{
+ HeapTuple indexTuple;
+ Form_pg_index indexForm;
+
+ indexTuple = SearchSysCacheCopy1(INDEXRELID,
+ ObjectIdGetDatum(indexOid));
+ if (!HeapTupleIsValid(indexTuple))
+ elog(ERROR, "cache lookup failed for index %u", indexOid);
+ indexForm = (Form_pg_index) GETSTRUCT(indexTuple);
+
+ /* this was checked earlier, but let's be real sure */
+ if (isclustered && !indexForm->indisvalid)
+ elog(ERROR, "cannot cluster on invalid index %u", indexOid);
+
+ indexForm->indisclustered = isclustered;
+ CatalogTupleUpdate(pg_index, &indexTuple->t_self, indexTuple);
+ heap_freetuple(indexTuple);
+}
+
/*
* mark_index_clustered: mark the specified index as the one clustered on
*
- * With indexOid == InvalidOid, will mark all indexes of rel not-clustered.
+ * With indexOid == InvalidOid, mark all indexes of rel not-clustered.
+ * Otherwise, mark children of the clustered index as clustered, and parents of
+ * other indexes as unclustered.
+ * We wish to maintain the following properties:
+ * 1) Only one index on a relation can be marked clustered at once
+ * 2) If a partitioned index is clustered, then all its children must be
+ * clustered.
*/
void
mark_index_clustered(Relation rel, Oid indexOid, bool is_internal)
{
- HeapTuple indexTuple;
- Form_pg_index indexForm;
- Relation pg_index;
- ListCell *index;
-
- /*
- * If the index is already marked clustered, no need to do anything.
- */
- if (OidIsValid(indexOid))
- {
- if (get_index_isclustered(indexOid))
- return;
- }
+ ListCell *lc, *lc2;
+ List *indexes;
+ Relation pg_index = table_open(IndexRelationId, RowExclusiveLock);
+ List *inh = find_all_inheritors(RelationGetRelid(rel), ShareRowExclusiveLock, NULL);
/*
* Check each index of the relation and set/clear the bit as needed.
+ * Iterate over the relation's children rather than the index's children
+ * since we need to unset cluster for indexes on intermediate children,
+ * too.
*/
- pg_index = table_open(IndexRelationId, RowExclusiveLock);
-
- foreach(index, RelationGetIndexList(rel))
+ foreach(lc, inh)
{
- Oid thisIndexOid = lfirst_oid(index);
-
- indexTuple = SearchSysCacheCopy1(INDEXRELID,
- ObjectIdGetDatum(thisIndexOid));
- if (!HeapTupleIsValid(indexTuple))
- elog(ERROR, "cache lookup failed for index %u", thisIndexOid);
- indexForm = (Form_pg_index) GETSTRUCT(indexTuple);
+ Oid inhrelid = lfirst_oid(lc);
+ Relation thisrel = table_open(inhrelid, ShareRowExclusiveLock);
- /*
- * Unset the bit if set. We know it's wrong because we checked this
- * earlier.
- */
- if (indexForm->indisclustered)
+ indexes = RelationGetIndexList(thisrel);
+ foreach (lc2, indexes)
{
- indexForm->indisclustered = false;
- CatalogTupleUpdate(pg_index, &indexTuple->t_self, indexTuple);
- }
- else if (thisIndexOid == indexOid)
- {
- /* this was checked earlier, but let's be real sure */
- if (!indexForm->indisvalid)
- elog(ERROR, "cannot cluster on invalid index %u", indexOid);
- indexForm->indisclustered = true;
- CatalogTupleUpdate(pg_index, &indexTuple->t_self, indexTuple);
- }
+ bool isclustered;
+ Oid thisIndexOid = lfirst_oid(lc2);
+ List *parentoids = get_rel_relispartition(thisIndexOid) ?
+ get_partition_ancestors(thisIndexOid) : NIL;
- InvokeObjectPostAlterHookArg(IndexRelationId, thisIndexOid, 0,
- InvalidOid, is_internal);
+ /*
+ * A child of the clustered index must be set clustered;
+ * indexes which are not children of the clustered index are
+ * set unclustered
+ */
+ isclustered = (thisIndexOid == indexOid) ||
+ list_member_oid(parentoids, indexOid);
+ Assert(OidIsValid(indexOid) || !isclustered);
+ set_indisclustered(thisIndexOid, isclustered, pg_index);
+
+ InvokeObjectPostAlterHookArg(IndexRelationId, thisIndexOid, 0,
+ InvalidOid, is_internal);
+ }
- heap_freetuple(indexTuple);
+ list_free(indexes);
+ table_close(thisrel, ShareRowExclusiveLock);
}
+ list_free(inh);
table_close(pg_index, RowExclusiveLock);
}
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 127ba7835d..4ca1ffbfa4 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -25,6 +25,7 @@
#include "catalog/catalog.h"
#include "catalog/index.h"
#include "catalog/indexing.h"
+#include "catalog/partition.h"
#include "catalog/pg_am.h"
#include "catalog/pg_constraint.h"
#include "catalog/pg_inherits.h"
@@ -32,6 +33,7 @@
#include "catalog/pg_opfamily.h"
#include "catalog/pg_tablespace.h"
#include "catalog/pg_type.h"
+#include "commands/cluster.h"
#include "commands/comment.h"
#include "commands/dbcommands.h"
#include "commands/defrem.h"
diff --git a/src/test/regress/expected/cluster.out b/src/test/regress/expected/cluster.out
index c74cfa88cc..1d436dfaae 100644
--- a/src/test/regress/expected/cluster.out
+++ b/src/test/regress/expected/cluster.out
@@ -495,6 +495,52 @@ Indexes:
"clstrpart_idx" btree (a) CLUSTER
Number of partitions: 3 (Use \d+ to list them.)
+-- Test that it recurses to grandchildren:
+\d clstrpart33
+ Table "public.clstrpart33"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+Partition of: clstrpart3 DEFAULT
+Indexes:
+ "clstrpart33_a_idx" btree (a) CLUSTER
+
+ALTER TABLE clstrpart SET WITHOUT CLUSTER;
+\d clstrpart33
+ Table "public.clstrpart33"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+Partition of: clstrpart3 DEFAULT
+Indexes:
+ "clstrpart33_a_idx" btree (a)
+
+ALTER TABLE clstrpart CLUSTER ON clstrpart_idx;
+\d clstrpart33
+ Table "public.clstrpart33"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+Partition of: clstrpart3 DEFAULT
+Indexes:
+ "clstrpart33_a_idx" btree (a) CLUSTER
+
+-- Check that only one child is marked clustered after marking clustered on a different parent
+CREATE INDEX clstrpart1_idx_2 ON clstrpart1(a);
+ALTER TABLE clstrpart CLUSTER ON clstrpart_idx;
+ALTER TABLE clstrpart1 CLUSTER ON clstrpart1_idx_2;
+\d clstrpart1
+ Partitioned table "public.clstrpart1"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+Partition of: clstrpart FOR VALUES FROM (1) TO (10)
+Partition key: RANGE (a)
+Indexes:
+ "clstrpart1_a_idx" btree (a)
+ "clstrpart1_idx_2" btree (a) CLUSTER
+Number of partitions: 2 (Use \d+ to list them.)
+
-- Test CLUSTER with external tuplesorting
create table clstr_4 as select * from tenk1;
create index cluster_sort on clstr_4 (hundred, thousand, tenthous);
diff --git a/src/test/regress/sql/cluster.sql b/src/test/regress/sql/cluster.sql
index 9bcc77695c..0ded2be1ca 100644
--- a/src/test/regress/sql/cluster.sql
+++ b/src/test/regress/sql/cluster.sql
@@ -220,6 +220,17 @@ CLUSTER clstrpart1 USING clstrpart1_a_idx; -- partition which is itself partitio
CLUSTER clstrpart12 USING clstrpart12_a_idx; -- partition which is itself partitioned, no childs
CLUSTER clstrpart2 USING clstrpart2_a_idx; -- leaf
\d clstrpart
+-- Test that it recurses to grandchildren:
+\d clstrpart33
+ALTER TABLE clstrpart SET WITHOUT CLUSTER;
+\d clstrpart33
+ALTER TABLE clstrpart CLUSTER ON clstrpart_idx;
+\d clstrpart33
+-- Check that only one child is marked clustered after marking clustered on a different parent
+CREATE INDEX clstrpart1_idx_2 ON clstrpart1(a);
+ALTER TABLE clstrpart CLUSTER ON clstrpart_idx;
+ALTER TABLE clstrpart1 CLUSTER ON clstrpart1_idx_2;
+\d clstrpart1
-- Test CLUSTER with external tuplesorting
--
2.17.0
--O5XBE6gyVG5Rl6Rj
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0005-Invalidate-parent-indexes.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-15 16:58 Robert Haas <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Robert Haas @ 2024-01-15 16:58 UTC (permalink / raw)
To: Alexander Lakhin <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Jakub Wartak <[email protected]>; Shinoda, Noriyoshi (HPE Services Japan - FSIP) <[email protected]>
On Sat, Jan 13, 2024 at 1:00 PM Alexander Lakhin <[email protected]> wrote:
> I've found one more typo in the sgml:
> summarized_pid
> And one in a comment:
> sumamry
>
> A trivial fix is attached.
Thanks, committed.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-15 20:31 Matthias van de Meent <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Matthias van de Meent @ 2024-01-15 20:31 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Mon, 15 Jan 2024 at 17:58, Robert Haas <[email protected]> wrote:
>
> On Sat, Jan 13, 2024 at 1:00 PM Alexander Lakhin <[email protected]> wrote:
> > I've found one more typo in the sgml:
> > summarized_pid
> > And one in a comment:
> > sumamry
> >
> > A trivial fix is attached.
>
> Thanks, committed.
Off-list I was notified that the new WAL summarizer process was not
yet added to the glossary, so PFA a patch that does that.
In passing, it also adds "incremental backup" to the glossary, and
updates the documented types of backends in monitoring.sgml with the
new backend type, too.
Kind regards,
Matthias van de Meent.
Attachments:
[application/octet-stream] v1-0001-incremental-backups-Add-new-items-to-glossary-mon.patch (4.2K, ../../CAEze2Who3iG9QXbV9gbWp3WvuwSh5D65Dzv9ktZEyxpRb_QvkQ@mail.gmail.com/2-v1-0001-incremental-backups-Add-new-items-to-glossary-mon.patch)
download | inline diff:
From 712dd139243c4ec8e48b4db1fa1109c3437081f3 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 15 Jan 2024 21:20:38 +0100
Subject: [PATCH v1] incremental backups: Add new items to glossary,
monitoring.sgml
The previous patches seem to have overlooked this.
---
doc/src/sgml/glossary.sgml | 37 ++++++++++++++++++++++++++++++++++++
doc/src/sgml/monitoring.sgml | 18 +++++++++++++++++-
2 files changed, 54 insertions(+), 1 deletion(-)
diff --git a/doc/src/sgml/glossary.sgml b/doc/src/sgml/glossary.sgml
index 5815fa4471..0f9d48a7dd 100644
--- a/doc/src/sgml/glossary.sgml
+++ b/doc/src/sgml/glossary.sgml
@@ -893,6 +893,29 @@
</para>
</glossdef>
</glossentry>
+ <glossentry id="glossary-incremental-backup">
+ <glossterm>Incremental Backup</glossterm>
+ <glossdef>
+ <para>
+ A backup that contains only the data that was potentially changed since
+ the previous <glossentry id="glossary-basebackup">base backup</glossentry>
+ and other incremental backups. It is often (but not always) smaller than
+ the WAL that generated the changes between the last basebackup and the
+ end of this incremental backup. Like base backups, it is generated by the
+ tool <xref linkend="app-pgbasebackup"/>.
+ </para>
+ <para>
+ In combination with <glossterm linkend="glossary-wal">WAL</glossterm>
+ data, recent incremental backups and a base backup this can restore a
+ <glossterm linkend="glossary-db-cluster"database cluster</glossterm> to
+ a consistent state without having to replay all the WAL that was generated
+ since the last base backup.
+ </para>
+ <para>
+ For more information, see <xref linkend="backup-incremental-backup"/>
+ </para>
+ </glossdef>
+ </glossentry>
<glossentry id="glossary-insert">
<glossterm>Insert</glossterm>
@@ -2157,6 +2180,20 @@
</glossdef>
</glossentry>
+ <glossentry id="glossary-wal-summarizer">
+ <glossterm>WAL summarizer (process)</glossterm>
+ <glossdef>
+ <para>
+ A special <glossterm linkend="glossary-backend">backend process</glossterm>
+ that summarizes WAL data for
+ <glossterm linkend="glossary-incremental-backup">incremental backups</glossterm>.
+ </para>
+ <para>
+ For more information, see <xref linkend="runtime-config-wal-summarization"/>
+ </para>
+ </glossdef>
+ </glossentry>
+
<glossentry id="glossary-wal-writer">
<glossterm>WAL writer (process)</glossterm>
<glossdef>
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index b804eb8b5e..6135d7a270 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -999,7 +999,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
<literal>client backend</literal>, <literal>checkpointer</literal>,
<literal>archiver</literal>, <literal>standalone backend</literal>,
<literal>startup</literal>, <literal>walreceiver</literal>,
- <literal>walsender</literal> and <literal>walwriter</literal>.
+ <literal>walsender</literal>, <literal>walwriter</literal> and
+ <literal>walsummarizer</literal>.
In addition, background workers registered by extensions may have
additional types.
</para></entry>
@@ -4062,6 +4063,21 @@ description | Waiting for a newly initialized WAL file to reach durable storage
</para>
</note>
+ <note>
+ <para>
+ Every time an index is searched, the index's
+ <structname>pg_stat_all_indexes</structname>.<structfield>idx_scan</structfield>
+ field is incremented. This usually happens once per index scan node
+ execution, but might take place several times during execution of a scan
+ that searches for multiple values together. Queries that use certain
+ <acronym>SQL</acronym> constructs to search for rows matching any value
+ out of a list (or an array) of multiple scalar values might perform
+ multiple <quote>primitive</quote> index scans (up to one primitive scan
+ per scalar value) at runtime. See <xref linkend="functions-comparisons"/>
+ for details.
+ </para>
+ </note>
+
</sect2>
<sect2 id="monitoring-pg-statio-all-tables-view">
--
2.40.1
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-16 15:39 Robert Haas <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Robert Haas @ 2024-01-16 15:39 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Mon, Jan 15, 2024 at 3:31 PM Matthias van de Meent
<[email protected]> wrote:
> Off-list I was notified that the new WAL summarizer process was not
> yet added to the glossary, so PFA a patch that does that.
> In passing, it also adds "incremental backup" to the glossary, and
> updates the documented types of backends in monitoring.sgml with the
> new backend type, too.
I wonder if it's possible that you sent the wrong version of this
patch, because:
(1) The docs don't build with this applied. I'm not sure if it's the
only problem, but <glossterm linkend="glossary-db-cluster" is missing
the closing >.
(2) The changes to monitoring.sgml contain an unrelated change, about
pg_stat_all_indexes.idx_scan.
Also, I think the "For more information, see <xref linkend="whatever"
/> bit should have a period after the markup tag, as we seem to do in
other cases.
One other thought is that the incremental backup only replaces
relation files with incremental files, and it never does anything
about FSM files. So the statement that it only contains data that was
potentially changed isn't quite correct. It might be better to phrase
it the other way around i.e. it is like a full backup, except that
some files can be replaced by incremental files which omit blocks to
which no WAL-logged changes have been made.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-16 20:22 Matthias van de Meent <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Matthias van de Meent @ 2024-01-16 20:22 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Tue, 16 Jan 2024 at 16:39, Robert Haas <[email protected]> wrote:
>
> On Mon, Jan 15, 2024 at 3:31 PM Matthias van de Meent
> <[email protected]> wrote:
> > Off-list I was notified that the new WAL summarizer process was not
> > yet added to the glossary, so PFA a patch that does that.
> > In passing, it also adds "incremental backup" to the glossary, and
> > updates the documented types of backends in monitoring.sgml with the
> > new backend type, too.
>
> I wonder if it's possible that you sent the wrong version of this
> patch, because:
>
> (1) The docs don't build with this applied. I'm not sure if it's the
> only problem, but <glossterm linkend="glossary-db-cluster" is missing
> the closing >.
That's my mistake, I didn't check install-world yet due to unrelated
issues building the docs. I've since sorted out these issues (this was
a good stick to get that done), so this issue is fixed in the attached
patch.
> (2) The changes to monitoring.sgml contain an unrelated change, about
> pg_stat_all_indexes.idx_scan.
Thanks for noticing, fixed in attached.
> Also, I think the "For more information, see <xref linkend="whatever"
> /> bit should have a period after the markup tag, as we seem to do in
> other cases.
Fixed.
> One other thought is that the incremental backup only replaces
> relation files with incremental files, and it never does anything
> about FSM files. So the statement that it only contains data that was
> potentially changed isn't quite correct. It might be better to phrase
> it the other way around i.e. it is like a full backup, except that
> some files can be replaced by incremental files which omit blocks to
> which no WAL-logged changes have been made.
How about the attached?
Kind regards,
Matthias van de Meent
Attachments:
[application/octet-stream] v2-0001-incremental-backups-Add-new-items-to-glossary-mon.patch (3.2K, ../../CAEze2WizrxyWg+xeVMuyVqfZc-8U1myy1V4H3zwV-7xXztVf2w@mail.gmail.com/2-v2-0001-incremental-backups-Add-new-items-to-glossary-mon.patch)
download | inline diff:
From 3d1092128c67c13004dff5a6ad58dbf35d05aa4d Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 15 Jan 2024 21:20:38 +0100
Subject: [PATCH v2] incremental backups: Add new items to glossary,
monitoring.sgml
The previous patches seem to have overlooked this.
---
doc/src/sgml/glossary.sgml | 36 ++++++++++++++++++++++++++++++++++++
doc/src/sgml/monitoring.sgml | 3 ++-
2 files changed, 38 insertions(+), 1 deletion(-)
diff --git a/doc/src/sgml/glossary.sgml b/doc/src/sgml/glossary.sgml
index 5815fa4471..2db0af766a 100644
--- a/doc/src/sgml/glossary.sgml
+++ b/doc/src/sgml/glossary.sgml
@@ -893,6 +893,28 @@
</para>
</glossdef>
</glossentry>
+ <glossentry id="glossary-incremental-backup">
+ <glossterm>Incremental Backup</glossterm>
+ <glossdef>
+ <para>
+ A special <glossterm linkend="glossary-basebackup">base backup</glossterm>
+ that for some WAL-logged relations only contains the pages that were
+ modified since a previous backup, as opposed to the full relation data of
+ normal base backups. Like base backups, it is generated by the tool
+ <xref linkend="app-pgbasebackup"/>.
+ </para>
+ <para>
+ To restore incremental backups the tool <xref linkend="app-pgcombinebackup"/>
+ is used, which combines the incremental backups with a base backup and
+ <glossterm linkend="glossary-wal">WAL</glossterm> to restore a
+ <glossterm linkend="glossary-db-cluster">database cluster</glossterm> to
+ a consistent state.
+ </para>
+ <para>
+ For more information, see <xref linkend="backup-incremental-backup"/>.
+ </para>
+ </glossdef>
+ </glossentry>
<glossentry id="glossary-insert">
<glossterm>Insert</glossterm>
@@ -2157,6 +2179,20 @@
</glossdef>
</glossentry>
+ <glossentry id="glossary-wal-summarizer">
+ <glossterm>WAL summarizer (process)</glossterm>
+ <glossdef>
+ <para>
+ A special <glossterm linkend="glossary-backend">backend process</glossterm>
+ that summarizes WAL data for
+ <glossterm linkend="glossary-incremental-backup">incremental backups</glossterm>.
+ </para>
+ <para>
+ For more information, see <xref linkend="runtime-config-wal-summarization"/>.
+ </para>
+ </glossdef>
+ </glossentry>
+
<glossentry id="glossary-wal-writer">
<glossterm>WAL writer (process)</glossterm>
<glossdef>
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index b804eb8b5e..6e74138a69 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -999,7 +999,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
<literal>client backend</literal>, <literal>checkpointer</literal>,
<literal>archiver</literal>, <literal>standalone backend</literal>,
<literal>startup</literal>, <literal>walreceiver</literal>,
- <literal>walsender</literal> and <literal>walwriter</literal>.
+ <literal>walsender</literal>, <literal>walwriter</literal> and
+ <literal>walsummarizer</literal>.
In addition, background workers registered by extensions may have
additional types.
</para></entry>
--
2.40.1
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-16 20:49 Robert Haas <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Robert Haas @ 2024-01-16 20:49 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Tue, Jan 16, 2024 at 3:22 PM Matthias van de Meent
<[email protected]> wrote:
> > One other thought is that the incremental backup only replaces
> > relation files with incremental files, and it never does anything
> > about FSM files. So the statement that it only contains data that was
> > potentially changed isn't quite correct. It might be better to phrase
> > it the other way around i.e. it is like a full backup, except that
> > some files can be replaced by incremental files which omit blocks to
> > which no WAL-logged changes have been made.
>
> How about the attached?
I like the direction.
+ A special <glossterm linkend="glossary-basebackup">base backup</glossterm>
+ that for some WAL-logged relations only contains the pages that were
+ modified since a previous backup, as opposed to the full relation data of
+ normal base backups. Like base backups, it is generated by the tool
+ <xref linkend="app-pgbasebackup"/>.
Could we say "that for some files may contain only those pages that
were modified since a previous backup, as opposed to the full contents
of every file"? My thoughts are (1) there's no hard guarantee that an
incremental backup will replace even one file with an incremental
file, although in practice it is probably almost always going to
happen and (2) pg_combinebackup would actually be totally fine with
any file at all being sent incrementally; it's only that the server
isn't smart enough to figure out how to do this with e.g. SLRU data
right now.
+ To restore incremental backups the tool <xref
linkend="app-pgcombinebackup"/>
+ is used, which combines the incremental backups with a base backup and
+ <glossterm linkend="glossary-wal">WAL</glossterm> to restore a
+ <glossterm linkend="glossary-db-cluster">database cluster</glossterm> to
+ a consistent state.
I wondered if this needed to be clearer that the chain of backups
could have length > 2. But on further reflection, I think it's fine,
unless you feel otherwise.
The rest LGTM.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-17 18:42 Matthias van de Meent <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Matthias van de Meent @ 2024-01-17 18:42 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Tue, 16 Jan 2024 at 21:49, Robert Haas <[email protected]> wrote:
>
> On Tue, Jan 16, 2024 at 3:22 PM Matthias van de Meent
> <[email protected]> wrote:
> + A special <glossterm linkend="glossary-basebackup">base backup</glossterm>
> + that for some WAL-logged relations only contains the pages that were
> + modified since a previous backup, as opposed to the full relation data of
> + normal base backups. Like base backups, it is generated by the tool
> + <xref linkend="app-pgbasebackup"/>.
>
> Could we say "that for some files may contain only those pages that
> were modified since a previous backup, as opposed to the full contents
> of every file"?
Sure, added in attached.
> + To restore incremental backups the tool <pgcombinebackup>
> + is used, which combines the incremental backups with a base backup and
> + [...]
> I wondered if this needed to be clearer that the chain of backups
> could have length > 2. But on further reflection, I think it's fine,
> unless you feel otherwise.
I removed "the" from the phrasing "the incremental backups", which
makes it a bit less restricted.
> The rest LGTM.
In the latest patch I also fixed the casing of "Incremental Backup" to
"... backup", to be in line with most other multi-word items.
Thanks!
Kind regards,
Matthias van de Meent
Neon (https://neon.tech)
Attachments:
[application/octet-stream] v3-0001-incremental-backups-Add-new-items-to-glossary-mon.patch (3.1K, ../../CAEze2Wjbf==0=v+Ck8B2vcCeqD_3_SfDV3HtpEQiP-r3e2fsSA@mail.gmail.com/2-v3-0001-incremental-backups-Add-new-items-to-glossary-mon.patch)
download | inline diff:
From 12a2b5290f29e3897391cc7b7a2666198cb1e835 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Mon, 15 Jan 2024 21:20:38 +0100
Subject: [PATCH v3] incremental backups: Add new items to glossary,
monitoring.sgml
The previous patches seem to have overlooked this.
---
doc/src/sgml/glossary.sgml | 35 +++++++++++++++++++++++++++++++++++
doc/src/sgml/monitoring.sgml | 3 ++-
2 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/doc/src/sgml/glossary.sgml b/doc/src/sgml/glossary.sgml
index 5815fa4471..a0150e5c48 100644
--- a/doc/src/sgml/glossary.sgml
+++ b/doc/src/sgml/glossary.sgml
@@ -893,6 +893,27 @@
</para>
</glossdef>
</glossentry>
+ <glossentry id="glossary-incremental-backup">
+ <glossterm>Incremental backup</glossterm>
+ <glossdef>
+ <para>
+ A special <glossterm linkend="glossary-basebackup">base backup</glossterm>
+ that for some files may contain only those pages that were modified since
+ a previous backup, as opposed to the full contents of every file. Like
+ base backups, it is generated by the tool <xref linkend="app-pgbasebackup"/>.
+ </para>
+ <para>
+ To restore incremental backups the tool <xref linkend="app-pgcombinebackup"/>
+ is used, which combines incremental backups with a base backup and
+ <glossterm linkend="glossary-wal">WAL</glossterm> to restore a
+ <glossterm linkend="glossary-db-cluster">database cluster</glossterm> to
+ a consistent state.
+ </para>
+ <para>
+ For more information, see <xref linkend="backup-incremental-backup"/>.
+ </para>
+ </glossdef>
+ </glossentry>
<glossentry id="glossary-insert">
<glossterm>Insert</glossterm>
@@ -2157,6 +2178,20 @@
</glossdef>
</glossentry>
+ <glossentry id="glossary-wal-summarizer">
+ <glossterm>WAL summarizer (process)</glossterm>
+ <glossdef>
+ <para>
+ A special <glossterm linkend="glossary-backend">backend process</glossterm>
+ that summarizes WAL data for
+ <glossterm linkend="glossary-incremental-backup">incremental backups</glossterm>.
+ </para>
+ <para>
+ For more information, see <xref linkend="runtime-config-wal-summarization"/>.
+ </para>
+ </glossdef>
+ </glossentry>
+
<glossentry id="glossary-wal-writer">
<glossterm>WAL writer (process)</glossterm>
<glossdef>
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index b804eb8b5e..6e74138a69 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -999,7 +999,8 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
<literal>client backend</literal>, <literal>checkpointer</literal>,
<literal>archiver</literal>, <literal>standalone backend</literal>,
<literal>startup</literal>, <literal>walreceiver</literal>,
- <literal>walsender</literal> and <literal>walwriter</literal>.
+ <literal>walsender</literal>, <literal>walwriter</literal> and
+ <literal>walsummarizer</literal>.
In addition, background workers registered by extensions may have
additional types.
</para></entry>
--
2.40.1
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-17 20:10 Robert Haas <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Robert Haas @ 2024-01-17 20:10 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Wed, Jan 17, 2024 at 1:42 PM Matthias van de Meent
<[email protected]> wrote:
> Sure, added in attached.
I think this mostly looks good now but I just realized that I think
this needs rephrasing:
+ To restore incremental backups the tool <xref
linkend="app-pgcombinebackup"/>
+ is used, which combines incremental backups with a base backup and
+ <glossterm linkend="glossary-wal">WAL</glossterm> to restore a
+ <glossterm linkend="glossary-db-cluster">database cluster</glossterm> to
+ a consistent state.
The way this is worded, at least to me, it makes it sound like
pg_combinebackup is going to do the WAL recovery for you, which it
isn't. Maybe:
To restore incremental backups the tool <xref
linkend="app-pgcombinebackup"/> is used, which combines incremental
backups with a base backup. Afterwards, recovery can use <glossterm
linkend="glossary-wal">WAL</glossterm> to bring the <glossterm
linkend="glossary-db-cluster">database cluster</glossterm> to a
consistent state.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-18 09:49 Matthias van de Meent <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Matthias van de Meent @ 2024-01-18 09:49 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Wed, 17 Jan 2024 at 21:10, Robert Haas <[email protected]> wrote:
>
> On Wed, Jan 17, 2024 at 1:42 PM Matthias van de Meent
> <[email protected]> wrote:
> > Sure, added in attached.
>
> I think this mostly looks good now but I just realized that I think
> this needs rephrasing:
>
> + To restore incremental backups the tool <xref
> linkend="app-pgcombinebackup"/>
> + is used, which combines incremental backups with a base backup and
> + <glossterm linkend="glossary-wal">WAL</glossterm> to restore a
> + <glossterm linkend="glossary-db-cluster">database cluster</glossterm> to
> + a consistent state.
>
> The way this is worded, at least to me, it makes it sound like
> pg_combinebackup is going to do the WAL recovery for you, which it
> isn't. Maybe:
>
> To restore incremental backups the tool <xref
> linkend="app-pgcombinebackup"/> is used, which combines incremental
> backups with a base backup. Afterwards, recovery can use <glossterm
> linkend="glossary-wal">WAL</glossterm> to bring the <glossterm
> linkend="glossary-db-cluster">database cluster</glossterm> to a
> consistent state.
Sure, that's fine with me.
Kind regards,
Matthias van de Meent
Neon (https://neon.tech)
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-18 14:40 Robert Haas <[email protected]>
parent: Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Robert Haas @ 2024-01-18 14:40 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Thu, Jan 18, 2024 at 4:50 AM Matthias van de Meent
<[email protected]> wrote:
> Sure, that's fine with me.
OK, committed that way.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-24 17:08 Nathan Bossart <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Nathan Bossart @ 2024-01-24 17:08 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
I'm seeing some recent buildfarm failures for pg_walsummary:
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=sungazer&dt=2024-01-14%2006%3A21%3A58
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=idiacanthus&dt=2024-01-17%2021%3A10%3A36
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=serinus&dt=2024-01-20%2018%3A58%3A49
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=taipan&dt=2024-01-23%2002%3A46%3A57
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=serinus&dt=2024-01-23%2020%3A23%3A36
The signature looks nearly identical in each:
# Failed test 'WAL summary file exists'
# at t/002_blocks.pl line 79.
# Failed test 'stdout shows block 0 modified'
# at t/002_blocks.pl line 85.
# ''
# doesn't match '(?^m:FORK main: block 0$)'
I haven't been able to reproduce the issue on my machine, and I haven't
figured out precisely what is happening yet, but I wanted to make sure
there is awareness.
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-24 17:46 Robert Haas <[email protected]>
parent: Nathan Bossart <[email protected]>
0 siblings, 2 replies; 33+ messages in thread
From: Robert Haas @ 2024-01-24 17:46 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jan 24, 2024 at 12:08 PM Nathan Bossart
<[email protected]> wrote:
> I'm seeing some recent buildfarm failures for pg_walsummary:
>
> https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=sungazer&dt=2024-01-14%2006%3A21%3A58
> https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=idiacanthus&dt=2024-01-17%2021%3A10%3A36
> https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=serinus&dt=2024-01-20%2018%3A58%3A49
> https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=taipan&dt=2024-01-23%2002%3A46%3A57
> https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=serinus&dt=2024-01-23%2020%3A23%3A36
>
> The signature looks nearly identical in each:
>
> # Failed test 'WAL summary file exists'
> # at t/002_blocks.pl line 79.
>
> # Failed test 'stdout shows block 0 modified'
> # at t/002_blocks.pl line 85.
> # ''
> # doesn't match '(?^m:FORK main: block 0$)'
>
> I haven't been able to reproduce the issue on my machine, and I haven't
> figured out precisely what is happening yet, but I wanted to make sure
> there is awareness.
This is weird. There's a little more detail in the log file,
regress_log_002_blocks, e.g. from the first failure you linked:
[11:18:20.683](96.787s) # before insert, summarized TLI 1 through 0/14E09D0
[11:18:21.188](0.505s) # after insert, summarized TLI 1 through 0/14E0D08
[11:18:21.326](0.138s) # examining summary for TLI 1 from 0/14E0D08 to 0/155BAF0
# 1
...
[11:18:21.349](0.000s) # got: 'pg_walsummary: error: could
not open file "/home/nm/farm/gcc64/HEAD/pgsql.build/src/bin/pg_walsummary/tmp_check/t_002_blocks_node1_data/pgdata/pg_wal/summaries/0000000100000000014E0D0800000000155BAF0
# 1.summary": No such file or directory'
The "examining summary" line is generated based on the output of
pg_available_wal_summaries(). The way that works is that the server
calls readdir(), disassembles the filename into a TLI and two LSNs,
and returns the result. Then, a fraction of a second later, the test
script reassembles those components into a filename and finds the file
missing. If the logic to translate between filenames and TLIs & LSNs
were incorrect, the test would fail consistently. So the only
explanation that seems to fit the facts is the file disappearing out
from under us. But that really shouldn't happen. We do have code to
remove such files in MaybeRemoveOldWalSummaries(), but it's only
supposed to be nuking files more than 10 days old.
So I don't really have a theory here as to what could be happening. :-(
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-24 18:05 Nathan Bossart <[email protected]>
parent: Robert Haas <[email protected]>
1 sibling, 2 replies; 33+ messages in thread
From: Nathan Bossart @ 2024-01-24 18:05 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jan 24, 2024 at 12:46:16PM -0500, Robert Haas wrote:
> The "examining summary" line is generated based on the output of
> pg_available_wal_summaries(). The way that works is that the server
> calls readdir(), disassembles the filename into a TLI and two LSNs,
> and returns the result. Then, a fraction of a second later, the test
> script reassembles those components into a filename and finds the file
> missing. If the logic to translate between filenames and TLIs & LSNs
> were incorrect, the test would fail consistently. So the only
> explanation that seems to fit the facts is the file disappearing out
> from under us. But that really shouldn't happen. We do have code to
> remove such files in MaybeRemoveOldWalSummaries(), but it's only
> supposed to be nuking files more than 10 days old.
>
> So I don't really have a theory here as to what could be happening. :-(
There might be an overflow risk in the cutoff time calculation, but I doubt
that's the root cause of these failures:
/*
* Files should only be removed if the last modification time precedes the
* cutoff time we compute here.
*/
cutoff_time = time(NULL) - 60 * wal_summary_keep_time;
Otherwise, I think we'll probably need to add some additional logging to
figure out what is happening...
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-24 19:08 Robert Haas <[email protected]>
parent: Nathan Bossart <[email protected]>
1 sibling, 1 reply; 33+ messages in thread
From: Robert Haas @ 2024-01-24 19:08 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jan 24, 2024 at 1:05 PM Nathan Bossart <[email protected]> wrote:
> There might be an overflow risk in the cutoff time calculation, but I doubt
> that's the root cause of these failures:
>
> /*
> * Files should only be removed if the last modification time precedes the
> * cutoff time we compute here.
> */
> cutoff_time = time(NULL) - 60 * wal_summary_keep_time;
>
> Otherwise, I think we'll probably need to add some additional logging to
> figure out what is happening...
Where, though? I suppose we could:
1. Change the server code so that it logs each WAL summary file
removed at a log level high enough to show up in the test logs.
2. Change the TAP test so that it prints out readdir(WAL summary
directory) at various points in the test.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-24 19:39 Nathan Bossart <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Nathan Bossart @ 2024-01-24 19:39 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jan 24, 2024 at 02:08:08PM -0500, Robert Haas wrote:
> On Wed, Jan 24, 2024 at 1:05 PM Nathan Bossart <[email protected]> wrote:
>> Otherwise, I think we'll probably need to add some additional logging to
>> figure out what is happening...
>
> Where, though? I suppose we could:
>
> 1. Change the server code so that it logs each WAL summary file
> removed at a log level high enough to show up in the test logs.
>
> 2. Change the TAP test so that it prints out readdir(WAL summary
> directory) at various points in the test.
That seems like a reasonable starting point. Even if it doesn't help
determine the root cause, it should at least help rule out concurrent
summary removal.
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-25 15:06 Robert Haas <[email protected]>
parent: Nathan Bossart <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Robert Haas @ 2024-01-25 15:06 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jan 24, 2024 at 2:39 PM Nathan Bossart <[email protected]> wrote:
> That seems like a reasonable starting point. Even if it doesn't help
> determine the root cause, it should at least help rule out concurrent
> summary removal.
Here is a patch for that.
--
Robert Haas
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] v1-0001-Temporary-patch-to-help-debug-pg_walsummary-test-.patch (3.4K, ../../CA+TgmoaC39=6Mc=b4zGVn86r7-1YPZtYQf=fVPaKdEXrA2i0dA@mail.gmail.com/2-v1-0001-Temporary-patch-to-help-debug-pg_walsummary-test-.patch)
download | inline diff:
From aa9a129de3a7860c9d1db239ee4c23940eb40759 Mon Sep 17 00:00:00 2001
From: Robert Haas <[email protected]>
Date: Thu, 25 Jan 2024 10:02:41 -0500
Subject: [PATCH v1] Temporary patch to help debug pg_walsummary test failures.
The tests in 002_blocks.pl are failing in the buildfarm from time to
time, but we don't know how to reproduce the failure elsewhere. The
most obvious explanation seems to be the unexpected disappearance of a
WAL summary file, so bump up the logging level in
RemoveWalSummaryIfOlderThan to try to help us spot such problems. Also
adjust 002_blocks.pl to dump out a directory listing of the relevant
directory at various points.
This patch should be reverted once we sort out what's happening here.
---
src/backend/backup/walsummary.c | 3 ++-
src/bin/pg_walsummary/t/002_blocks.pl | 14 ++++++++++++++
2 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/src/backend/backup/walsummary.c b/src/backend/backup/walsummary.c
index b549673a9d..bef75a9b47 100644
--- a/src/backend/backup/walsummary.c
+++ b/src/backend/backup/walsummary.c
@@ -251,7 +251,8 @@ RemoveWalSummaryIfOlderThan(WalSummaryFile *ws, time_t cutoff_time)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not stat file \"%s\": %m", path)));
- ereport(DEBUG2,
+ /* XXX: was DEBUG2, temporarily increased to LOG */
+ ereport(LOG,
(errmsg_internal("removing file \"%s\"", path)));
}
diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl
index d609d2c547..40908da8cb 100644
--- a/src/bin/pg_walsummary/t/002_blocks.pl
+++ b/src/bin/pg_walsummary/t/002_blocks.pl
@@ -48,6 +48,7 @@ SELECT summarized_tli, summarized_lsn FROM pg_get_wal_summarizer_state()
EOM
($summarized_tli, $summarized_lsn) = split(/\|/, $progress);
note("after insert, summarized TLI $summarized_tli through $summarized_lsn");
+note_wal_summary_dir("after insert", $node1);
# Update a row in the first block of the table and trigger a checkpoint.
$node1->safe_psql('postgres', <<EOM);
@@ -70,6 +71,7 @@ SELECT tli, start_lsn, end_lsn from pg_available_wal_summaries()
EOM
my ($tli, $start_lsn, $end_lsn) = split(/\|/, $details);
note("examining summary for TLI $tli from $start_lsn to $end_lsn");
+note_wal_summary_dir("after new summary", $node1);
# Reconstruct the full pathname for the WAL summary file.
my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
@@ -77,6 +79,7 @@ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
split(m@/@, $start_lsn),
split(m@/@, $end_lsn);
ok(-f $filename, "WAL summary file exists");
+note_wal_summary_dir("after existence check", $node1);
# Run pg_walsummary on it. We expect block 0 to be modified, but depending
# on where the new tuple ends up, block 1 might also be modified, so we
@@ -84,5 +87,16 @@ ok(-f $filename, "WAL summary file exists");
my ($stdout, $stderr) = run_command([ 'pg_walsummary', '-i', $filename ]);
like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified");
is($stderr, '', 'stderr is empty');
+note_wal_summary_dir("after pg_walsummary run", $node1);
done_testing();
+
+# XXX. Temporary debugging code.
+sub note_wal_summary_dir
+{
+ my ($flair, $node) = @_;
+
+ my $wsdir = sprintf "%s/pg_wal/summaries", $node->data_dir;
+ my @wsfiles = grep { $_ ne '.' && $_ ne '..' } slurp_dir($wsdir);
+ note("$flair pg_wal/summaries has: @wsfiles");
+}
--
2.39.3 (Apple Git-145)
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-25 16:08 Nathan Bossart <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Nathan Bossart @ 2024-01-25 16:08 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Thu, Jan 25, 2024 at 10:06:41AM -0500, Robert Haas wrote:
> On Wed, Jan 24, 2024 at 2:39 PM Nathan Bossart <[email protected]> wrote:
>> That seems like a reasonable starting point. Even if it doesn't help
>> determine the root cause, it should at least help rule out concurrent
>> summary removal.
>
> Here is a patch for that.
LGTM. The only thing I might add is the cutoff_time in that LOG.
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-26 16:04 Robert Haas <[email protected]>
parent: Nathan Bossart <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Robert Haas @ 2024-01-26 16:04 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Thu, Jan 25, 2024 at 11:08 AM Nathan Bossart
<[email protected]> wrote:
> On Thu, Jan 25, 2024 at 10:06:41AM -0500, Robert Haas wrote:
> > On Wed, Jan 24, 2024 at 2:39 PM Nathan Bossart <[email protected]> wrote:
> >> That seems like a reasonable starting point. Even if it doesn't help
> >> determine the root cause, it should at least help rule out concurrent
> >> summary removal.
> >
> > Here is a patch for that.
>
> LGTM. The only thing I might add is the cutoff_time in that LOG.
Here is v2 with that addition.
--
Robert Haas
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] v2-0001-Temporary-patch-to-help-debug-pg_walsummary-test-.patch (3.6K, ../../CA+TgmoZLXsorAX2XNxC1+zKFSagCZUk4Qiq49+WxTk8oePvWCQ@mail.gmail.com/2-v2-0001-Temporary-patch-to-help-debug-pg_walsummary-test-.patch)
download | inline diff:
From c79461a1147acc851e80c5b5885e7a1d3b591050 Mon Sep 17 00:00:00 2001
From: Robert Haas <[email protected]>
Date: Fri, 26 Jan 2024 11:03:29 -0500
Subject: [PATCH v2] Temporary patch to help debug pg_walsummary test failures.
The tests in 002_blocks.pl are failing in the buildfarm from time to
time, but we don't know how to reproduce the failure elsewhere. The
most obvious explanation seems to be the unexpected disappearance of a
WAL summary file, so bump up the logging level in
RemoveWalSummaryIfOlderThan to try to help us spot such problems, and
print the cutoff time in addition to the removed filename. Also
adjust 002_blocks.pl to dump out a directory listing of the relevant
directory at various points.
This patch should be reverted once we sort out what's happening here.
---
src/backend/backup/walsummary.c | 7 +++++++
src/bin/pg_walsummary/t/002_blocks.pl | 14 ++++++++++++++
2 files changed, 21 insertions(+)
diff --git a/src/backend/backup/walsummary.c b/src/backend/backup/walsummary.c
index b549673a9d..ae314d8b74 100644
--- a/src/backend/backup/walsummary.c
+++ b/src/backend/backup/walsummary.c
@@ -251,8 +251,15 @@ RemoveWalSummaryIfOlderThan(WalSummaryFile *ws, time_t cutoff_time)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not stat file \"%s\": %m", path)));
+ /* XXX temporarily changed to debug buildfarm failures */
+#if 0
ereport(DEBUG2,
(errmsg_internal("removing file \"%s\"", path)));
+#else
+ ereport(LOG,
+ (errmsg_internal("removing file \"%s\" cutoff_time=%llu", path,
+ (unsigned long long) cutoff_time)));
+#endif
}
/*
diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl
index d609d2c547..40908da8cb 100644
--- a/src/bin/pg_walsummary/t/002_blocks.pl
+++ b/src/bin/pg_walsummary/t/002_blocks.pl
@@ -48,6 +48,7 @@ SELECT summarized_tli, summarized_lsn FROM pg_get_wal_summarizer_state()
EOM
($summarized_tli, $summarized_lsn) = split(/\|/, $progress);
note("after insert, summarized TLI $summarized_tli through $summarized_lsn");
+note_wal_summary_dir("after insert", $node1);
# Update a row in the first block of the table and trigger a checkpoint.
$node1->safe_psql('postgres', <<EOM);
@@ -70,6 +71,7 @@ SELECT tli, start_lsn, end_lsn from pg_available_wal_summaries()
EOM
my ($tli, $start_lsn, $end_lsn) = split(/\|/, $details);
note("examining summary for TLI $tli from $start_lsn to $end_lsn");
+note_wal_summary_dir("after new summary", $node1);
# Reconstruct the full pathname for the WAL summary file.
my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
@@ -77,6 +79,7 @@ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
split(m@/@, $start_lsn),
split(m@/@, $end_lsn);
ok(-f $filename, "WAL summary file exists");
+note_wal_summary_dir("after existence check", $node1);
# Run pg_walsummary on it. We expect block 0 to be modified, but depending
# on where the new tuple ends up, block 1 might also be modified, so we
@@ -84,5 +87,16 @@ ok(-f $filename, "WAL summary file exists");
my ($stdout, $stderr) = run_command([ 'pg_walsummary', '-i', $filename ]);
like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified");
is($stderr, '', 'stderr is empty');
+note_wal_summary_dir("after pg_walsummary run", $node1);
done_testing();
+
+# XXX. Temporary debugging code.
+sub note_wal_summary_dir
+{
+ my ($flair, $node) = @_;
+
+ my $wsdir = sprintf "%s/pg_wal/summaries", $node->data_dir;
+ my @wsfiles = grep { $_ ne '.' && $_ ne '..' } slurp_dir($wsdir);
+ note("$flair pg_wal/summaries has: @wsfiles");
+}
--
2.39.3 (Apple Git-145)
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-26 17:39 Nathan Bossart <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Nathan Bossart @ 2024-01-26 17:39 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Jan 26, 2024 at 11:04:37AM -0500, Robert Haas wrote:
> Here is v2 with that addition.
Looks reasonable.
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-26 18:37 Robert Haas <[email protected]>
parent: Nathan Bossart <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Robert Haas @ 2024-01-26 18:37 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Jan 26, 2024 at 12:39 PM Nathan Bossart
<[email protected]> wrote:
> On Fri, Jan 26, 2024 at 11:04:37AM -0500, Robert Haas wrote:
> > Here is v2 with that addition.
>
> Looks reasonable.
Thanks for the report & review. I have committed that version.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-27 07:00 Alexander Lakhin <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Alexander Lakhin @ 2024-01-27 07:00 UTC (permalink / raw)
To: Robert Haas <[email protected]>; Nathan Bossart <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
Hello Robert,
26.01.2024 21:37, Robert Haas wrote:
> On Fri, Jan 26, 2024 at 12:39 PM Nathan Bossart
> <[email protected]> wrote:
>> On Fri, Jan 26, 2024 at 11:04:37AM -0500, Robert Haas wrote:
>>> Here is v2 with that addition.
>> Looks reasonable.
> Thanks for the report & review. I have committed that version.
While trying to reproduce the 002_blocks test failure, I've encountered
another anomaly (or two):
make -s check -C src/bin/pg_walsummary/ PROVE_TESTS="t/002*" PROVE_FLAGS="--timer"
# +++ tap check in src/bin/pg_walsummary +++
[05:40:38] t/002_blocks.pl .. # poll_query_until timed out executing this query:
# SELECT EXISTS (
# SELECT * from pg_available_wal_summaries()
# WHERE tli = 0 AND end_lsn > '0/0'
# )
#
# expecting this output:
# t
# last actual query output:
# f
# with stderr:
[05:40:38] t/002_blocks.pl .. ok 266739 ms ( 0.00 usr 0.01 sys + 17.51 cusr 26.79 csys = 44.31 CPU)
[05:45:05]
All tests successful.
Files=1, Tests=3, 267 wallclock secs ( 0.02 usr 0.02 sys + 17.51 cusr 26.79 csys = 44.34 CPU)
Result: PASS
It looks like the test may call pg_get_wal_summarizer_state() when
WalSummarizerCtl->initialized is false yet, i. e. before the first
GetOldestUnsummarizedLSN() call.
I could reproduce the issue easily (within 10 test runs) with
pg_usleep(100000L);
added inside WalSummarizerMain() just below:
sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
But the fact that the test passes regardless of the timeout, make me
wonder, whether any test should fail when such timeout occurs?
Best regards,
Alexander
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-27 08:00 Alexander Lakhin <[email protected]>
parent: Robert Haas <[email protected]>
1 sibling, 1 reply; 33+ messages in thread
From: Alexander Lakhin @ 2024-01-27 08:00 UTC (permalink / raw)
To: Robert Haas <[email protected]>; Nathan Bossart <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
24.01.2024 20:46, Robert Haas wrote:
> This is weird. There's a little more detail in the log file,
> regress_log_002_blocks, e.g. from the first failure you linked:
>
> [11:18:20.683](96.787s) # before insert, summarized TLI 1 through 0/14E09D0
> [11:18:21.188](0.505s) # after insert, summarized TLI 1 through 0/14E0D08
> [11:18:21.326](0.138s) # examining summary for TLI 1 from 0/14E0D08 to 0/155BAF0
> # 1
> ...
> [11:18:21.349](0.000s) # got: 'pg_walsummary: error: could
> not open file "/home/nm/farm/gcc64/HEAD/pgsql.build/src/bin/pg_walsummary/tmp_check/t_002_blocks_node1_data/pgdata/pg_wal/summaries/0000000100000000014E0D0800000000155BAF0
> # 1.summary": No such file or directory'
>
> The "examining summary" line is generated based on the output of
> pg_available_wal_summaries(). The way that works is that the server
> calls readdir(), disassembles the filename into a TLI and two LSNs,
> and returns the result.
I'm discouraged by "\n1" in the file name and in the
"examining summary..." message.
regress_log_002_blocks from the following successful test run on the same
sungazer node contains:
[15:21:58.924](0.106s) # examining summary for TLI 1 from 0/155BAE0 to 0/155E750
[15:21:58.925](0.001s) ok 1 - WAL summary file exists
Best regards,
Alexander
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-27 16:31 Nathan Bossart <[email protected]>
parent: Alexander Lakhin <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Nathan Bossart @ 2024-01-27 16:31 UTC (permalink / raw)
To: Alexander Lakhin <[email protected]>; +Cc: Robert Haas <[email protected]>; Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Sat, Jan 27, 2024 at 11:00:01AM +0300, Alexander Lakhin wrote:
> 24.01.2024 20:46, Robert Haas wrote:
>> This is weird. There's a little more detail in the log file,
>> regress_log_002_blocks, e.g. from the first failure you linked:
>>
>> [11:18:20.683](96.787s) # before insert, summarized TLI 1 through 0/14E09D0
>> [11:18:21.188](0.505s) # after insert, summarized TLI 1 through 0/14E0D08
>> [11:18:21.326](0.138s) # examining summary for TLI 1 from 0/14E0D08 to 0/155BAF0
>> # 1
>> ...
>> [11:18:21.349](0.000s) # got: 'pg_walsummary: error: could
>> not open file "/home/nm/farm/gcc64/HEAD/pgsql.build/src/bin/pg_walsummary/tmp_check/t_002_blocks_node1_data/pgdata/pg_wal/summaries/0000000100000000014E0D0800000000155BAF0
>> # 1.summary": No such file or directory'
>>
>> The "examining summary" line is generated based on the output of
>> pg_available_wal_summaries(). The way that works is that the server
>> calls readdir(), disassembles the filename into a TLI and two LSNs,
>> and returns the result.
>
> I'm discouraged by "\n1" in the file name and in the
> "examining summary..." message.
> regress_log_002_blocks from the following successful test run on the same
> sungazer node contains:
> [15:21:58.924](0.106s) # examining summary for TLI 1 from 0/155BAE0 to 0/155E750
> [15:21:58.925](0.001s) ok 1 - WAL summary file exists
Ah, I think this query:
SELECT tli, start_lsn, end_lsn from pg_available_wal_summaries()
WHERE tli = $summarized_tli AND end_lsn > '$summarized_lsn'
is returning more than one row in some cases. I attached a quick sketch of
an easy way to reproduce the issue as well as one way to fix it.
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
Attachments:
[text/x-diff] repro_and_fix.patch (897B, ../../20240127163109.GA3047116@nathanxps13/2-repro_and_fix.patch)
download | inline diff:
diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl
index 40908da8cb..5609b1bd14 100644
--- a/src/bin/pg_walsummary/t/002_blocks.pl
+++ b/src/bin/pg_walsummary/t/002_blocks.pl
@@ -64,10 +64,16 @@ SELECT EXISTS (
)
EOM
+$node1->safe_psql('postgres', <<EOM);
+UPDATE mytable SET b = 'abefghijklmnopqrstuvwxyz' WHERE a = 2;
+CHECKPOINT;
+EOM
+$node1->safe_psql('postgres', 'SELECT pg_sleep(1);');
+
# Figure out the exact details for the new summary file.
my $details = $node1->safe_psql('postgres', <<EOM);
SELECT tli, start_lsn, end_lsn from pg_available_wal_summaries()
- WHERE tli = $summarized_tli AND end_lsn > '$summarized_lsn'
+ WHERE tli = $summarized_tli AND end_lsn > '$summarized_lsn' ORDER BY end_lsn LIMIT 1
EOM
my ($tli, $start_lsn, $end_lsn) = split(/\|/, $details);
note("examining summary for TLI $tli from $start_lsn to $end_lsn");
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-29 18:21 Nathan Bossart <[email protected]>
parent: Nathan Bossart <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Nathan Bossart @ 2024-01-29 18:21 UTC (permalink / raw)
To: Alexander Lakhin <[email protected]>; +Cc: Robert Haas <[email protected]>; Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Sat, Jan 27, 2024 at 10:31:09AM -0600, Nathan Bossart wrote:
> On Sat, Jan 27, 2024 at 11:00:01AM +0300, Alexander Lakhin wrote:
>> I'm discouraged by "\n1" in the file name and in the
>> "examining summary..." message.
>> regress_log_002_blocks from the following successful test run on the same
>> sungazer node contains:
>> [15:21:58.924](0.106s) # examining summary for TLI 1 from 0/155BAE0 to 0/155E750
>> [15:21:58.925](0.001s) ok 1 - WAL summary file exists
>
> Ah, I think this query:
>
> SELECT tli, start_lsn, end_lsn from pg_available_wal_summaries()
> WHERE tli = $summarized_tli AND end_lsn > '$summarized_lsn'
>
> is returning more than one row in some cases. I attached a quick sketch of
> an easy way to reproduce the issue as well as one way to fix it.
The buildfarm just caught a failure with the new logging in place:
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=calliphoridae&dt=2024-01-29%2018%3A09%3A...
I'm not totally sure my "fix" is correct, but I think this does confirm the
theory.
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-29 20:18 Robert Haas <[email protected]>
parent: Nathan Bossart <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Robert Haas @ 2024-01-29 20:18 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Alexander Lakhin <[email protected]>; Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, Jan 29, 2024 at 1:21 PM Nathan Bossart <[email protected]> wrote:
> > Ah, I think this query:
> >
> > SELECT tli, start_lsn, end_lsn from pg_available_wal_summaries()
> > WHERE tli = $summarized_tli AND end_lsn > '$summarized_lsn'
> >
> > is returning more than one row in some cases. I attached a quick sketch of
> > an easy way to reproduce the issue as well as one way to fix it.
>
> The buildfarm just caught a failure with the new logging in place:
>
> https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=calliphoridae&dt=2024-01-29%2018%3A09%3A...
>
> I'm not totally sure my "fix" is correct, but I think this does confirm the
> theory.
Ah. The possibilities of ending up with TWO new WAL summaries never
occurred to me. Things that never occurred to the developer are a
leading cause of bugs, and so here.
I'm wondering if what we need to do is run pg_walsummary on both
summary files in that case. If we just pick one or the other, how do
we know which one to pick?
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-29 21:13 Nathan Bossart <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Nathan Bossart @ 2024-01-29 21:13 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Alexander Lakhin <[email protected]>; Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, Jan 29, 2024 at 03:18:50PM -0500, Robert Haas wrote:
> I'm wondering if what we need to do is run pg_walsummary on both
> summary files in that case. If we just pick one or the other, how do
> we know which one to pick?
Even if we do that, isn't it possible that none of the summaries will
include the change? Presently, we get the latest summarized LSN, make a
change, and then wait for the next summary file with a greater LSN than
what we saw before the change. But AFAICT there's no guarantee that means
the change has been summarized yet, although the chances of that happening
in a test are probably pretty small.
Could we get the LSN before and after making the change and then inspect
all summaries that include that LSN range?
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-30 15:51 Robert Haas <[email protected]>
parent: Nathan Bossart <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Robert Haas @ 2024-01-30 15:51 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Alexander Lakhin <[email protected]>; Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, Jan 29, 2024 at 4:13 PM Nathan Bossart <[email protected]> wrote:
> On Mon, Jan 29, 2024 at 03:18:50PM -0500, Robert Haas wrote:
> > I'm wondering if what we need to do is run pg_walsummary on both
> > summary files in that case. If we just pick one or the other, how do
> > we know which one to pick?
>
> Even if we do that, isn't it possible that none of the summaries will
> include the change? Presently, we get the latest summarized LSN, make a
> change, and then wait for the next summary file with a greater LSN than
> what we saw before the change. But AFAICT there's no guarantee that means
> the change has been summarized yet, although the chances of that happening
> in a test are probably pretty small.
>
> Could we get the LSN before and after making the change and then inspect
> all summaries that include that LSN range?
The trick here is that each WAL summary file covers one checkpoint
cycle. The intent of the test is to load data into the table,
checkpoint, see what summaries exist, then update a row, checkpoint
again, and see what summaries now exist. We expect one new summary
because there's been one new checkpoint. When I was thinking about
this yesterday, I was imagining that we were somehow getting an extra
checkpoint in some cases. But it looks like it's actually an
off-by-one situation. In
https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=calliphoridae&dt=2024-01-29%2018%3A09%3A...
the new files that show up between "after insert" and "after new
summary" are:
00000001000000000152FAE000000000015AAAC8.summary (LSN distance ~500k)
00000001000000000152F7A8000000000152FAE0.summary (LSN distance 824 bytes)
The checkpoint after the inserts says:
LOG: checkpoint complete: wrote 14 buffers (10.9%); 0 WAL file(s)
added, 0 removed, 0 recycled; write=0.956 s, sync=0.929 s, total=3.059
s; sync files=39, longest=0.373 s, average=0.024 s; distance=491 kB,
estimate=491 kB; lsn=0/15AAB20, redo lsn=0/15AAAC8
And the checkpoint after the single-row update says:
LOG: checkpoint complete: wrote 4 buffers (3.1%); 0 WAL file(s)
added, 0 removed, 0 recycled; write=0.648 s, sync=0.355 s, total=2.798
s; sync files=3, longest=0.348 s, average=0.119 s; distance=11 kB,
estimate=443 kB; lsn=0/15AD770, redo lsn=0/15AD718
So both of the new WAL summary files that are appearing here are from
checkpoints that happened before the single-row update. The larger
file is the one covering the 400 inserts, and the smaller one is the
checkpoint before that. Which means that the "Wait for a new summary
to show up." code isn't actually waiting long enough, and then the
whole thing goes haywire. The problem is, I think, that this code
naively thinks it can just wait for summarized_lsn and everything will
be fine ... but that assumes we were caught up when we first measured
the summarized_lsn, and that need not be so, because it takes some
short but non-zero amount of time for the summarizer to catch up with
the WAL generated during initdb.
I think the solution here is to find a better way to wait for the
inserts to be summarized, one that actually does wait for that to
happen.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-30 16:52 Robert Haas <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Robert Haas @ 2024-01-30 16:52 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Alexander Lakhin <[email protected]>; Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, Jan 30, 2024 at 10:51 AM Robert Haas <[email protected]> wrote:
> I think the solution here is to find a better way to wait for the
> inserts to be summarized, one that actually does wait for that to
> happen.
Here's a patch for that. I now think
a7097ca630a25dd2248229f21ebce4968d85d10a was actually misguided, and
served only to mask some of the failures caused by waiting for the WAL
summary file.
--
Robert Haas
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] v1-0001-Revise-pg_walsummary-s-002_blocks-test-to-avoid-s.patch (4.7K, ../../CA+TgmobWFb8NqyfC31YnKAbZiXf9tLuwmyuvx=iYMXMniPQ4nw@mail.gmail.com/2-v1-0001-Revise-pg_walsummary-s-002_blocks-test-to-avoid-s.patch)
download | inline diff:
From 890160b2760ac348af5edae7e6af07318b7fb20e Mon Sep 17 00:00:00 2001
From: Robert Haas <[email protected]>
Date: Tue, 30 Jan 2024 11:48:58 -0500
Subject: [PATCH v1] Revise pg_walsummary's 002_blocks test to avoid spurious
failures.
Analysis of buildfarm results showed that the code that was intended
to wait for the inserts performed by this test to complete did not
actually do so. Try to make that logic more robust.
Improve error checking elsewhere in the script, too, so that we
don't miss things like poll_query_until failing.
---
src/bin/pg_walsummary/t/002_blocks.pl | 47 ++++++++++++++++-----------
1 file changed, 28 insertions(+), 19 deletions(-)
diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl
index 40908da8cb..eb128f460c 100644
--- a/src/bin/pg_walsummary/t/002_blocks.pl
+++ b/src/bin/pg_walsummary/t/002_blocks.pl
@@ -13,13 +13,6 @@ $node1->init(has_archiving => 1, allows_streaming => 1);
$node1->append_conf('postgresql.conf', 'summarize_wal = on');
$node1->start;
-# See what's been summarized up until now.
-my $progress = $node1->safe_psql('postgres', <<EOM);
-SELECT summarized_tli, summarized_lsn FROM pg_get_wal_summarizer_state()
-EOM
-my ($summarized_tli, $summarized_lsn) = split(/\|/, $progress);
-note("before insert, summarized TLI $summarized_tli through $summarized_lsn");
-
# Create a table and insert a few test rows into it. VACUUM FREEZE it so that
# autovacuum doesn't induce any future modifications unexpectedly. Then
# trigger a checkpoint.
@@ -31,22 +24,33 @@ SELECT
FROM
generate_series(1, 400) g;
VACUUM FREEZE;
+EOM
+
+# Record the current WAL insert LSN.
+my $base_lsn = $node1->safe_psql('postgres', <<EOM);
+SELECT pg_current_wal_insert_lsn()
+EOM
+note("just after insert, WAL insert LSN is $base_lsn");
+
+# Now perform a CHECKPOINT.
+$node1->safe_psql('postgres', <<EOM);
CHECKPOINT;
EOM
-# Wait for a new summary to show up.
-$node1->poll_query_until('postgres', <<EOM);
+# Wait for a new summary to show up, one that includes the inserts we just did.
+my $result = $node1->poll_query_until('postgres', <<EOM);
SELECT EXISTS (
SELECT * from pg_available_wal_summaries()
- WHERE tli = $summarized_tli AND end_lsn > '$summarized_lsn'
+ WHERE end_lsn >= '$base_lsn'
)
EOM
+ok($result, "WAL summarization caught up after insert");
-# Again check the progress of WAL summarization.
-$progress = $node1->safe_psql('postgres', <<EOM);
+# Get a list of what summaries we now have.
+my $progress = $node1->safe_psql('postgres', <<EOM);
SELECT summarized_tli, summarized_lsn FROM pg_get_wal_summarizer_state()
EOM
-($summarized_tli, $summarized_lsn) = split(/\|/, $progress);
+my ($summarized_tli, $summarized_lsn) = split(/\|/, $progress);
note("after insert, summarized TLI $summarized_tli through $summarized_lsn");
note_wal_summary_dir("after insert", $node1);
@@ -56,20 +60,23 @@ UPDATE mytable SET b = 'abcdefghijklmnopqrstuvwxyz' WHERE a = 2;
CHECKPOINT;
EOM
-# Again wait for a new summary to show up.
-$node1->poll_query_until('postgres', <<EOM);
+# Wait for a new summary to show up.
+$result = $node1->poll_query_until('postgres', <<EOM);
SELECT EXISTS (
SELECT * from pg_available_wal_summaries()
WHERE tli = $summarized_tli AND end_lsn > '$summarized_lsn'
)
EOM
+ok($result, "got new WAL summary after update");
# Figure out the exact details for the new summary file.
my $details = $node1->safe_psql('postgres', <<EOM);
SELECT tli, start_lsn, end_lsn from pg_available_wal_summaries()
WHERE tli = $summarized_tli AND end_lsn > '$summarized_lsn'
EOM
-my ($tli, $start_lsn, $end_lsn) = split(/\|/, $details);
+my @lines = split(/\n/, $details);
+is(0+@lines, 1, "got exactly one new WAL summary");
+my ($tli, $start_lsn, $end_lsn) = split(/\|/, $lines[0]);
note("examining summary for TLI $tli from $start_lsn to $end_lsn");
note_wal_summary_dir("after new summary", $node1);
@@ -81,12 +88,14 @@ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
ok(-f $filename, "WAL summary file exists");
note_wal_summary_dir("after existence check", $node1);
-# Run pg_walsummary on it. We expect block 0 to be modified, but depending
-# on where the new tuple ends up, block 1 might also be modified, so we
-# pass -i to pg_walsummary to make sure we don't end up with a 0..1 range.
+# Run pg_walsummary on it. We expect exactly two blocks to be modified,
+# block 0 and one other.
my ($stdout, $stderr) = run_command([ 'pg_walsummary', '-i', $filename ]);
+note($stdout);
+@lines = split(/\n/, $stdout);
like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified");
is($stderr, '', 'stderr is empty');
+is(0+@lines, 2, "UPDATE modified 2 blocks");
note_wal_summary_dir("after pg_walsummary run", $node1);
done_testing();
--
2.39.3 (Apple Git-145)
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-01-31 15:33 Robert Haas <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Robert Haas @ 2024-01-31 15:33 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Alexander Lakhin <[email protected]>; Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, Jan 30, 2024 at 11:52 AM Robert Haas <[email protected]> wrote:
> Here's a patch for that. I now think
> a7097ca630a25dd2248229f21ebce4968d85d10a was actually misguided, and
> served only to mask some of the failures caused by waiting for the WAL
> summary file.
Committed.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-03-14 21:00 Nathan Bossart <[email protected]>
parent: Nathan Bossart <[email protected]>
1 sibling, 1 reply; 33+ messages in thread
From: Nathan Bossart @ 2024-03-14 21:00 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jan 24, 2024 at 12:05:15PM -0600, Nathan Bossart wrote:
> There might be an overflow risk in the cutoff time calculation, but I doubt
> that's the root cause of these failures:
>
> /*
> * Files should only be removed if the last modification time precedes the
> * cutoff time we compute here.
> */
> cutoff_time = time(NULL) - 60 * wal_summary_keep_time;
I've attached a short patch for fixing this overflow risk. Specifically,
it limits wal_summary_keep_time to INT_MAX / SECS_PER_MINUTE, just like
log_rotation_age.
I considering checking for overflow when we subtract the keep-time from the
result of time(2), but AFAICT there's only a problem if time_t is unsigned,
which Wikipedia leads me to believe is unusual [0], so I figured we might
be able to just wait this one out until 2038.
> Otherwise, I think we'll probably need to add some additional logging to
> figure out what is happening...
Separately, I suppose it's probably time to revert the temporary debugging
code adding by commit 5ddf997. I can craft a patch for that, too.
[0] https://en.wikipedia.org/wiki/Unix_time#Representing_the_number
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
Attachments:
[text/x-diff] overflow_fix.patch (1.5K, ../../20240314210010.GA3056455@nathanxps13/2-overflow_fix.patch)
download | inline diff:
diff --git a/src/backend/postmaster/walsummarizer.c b/src/backend/postmaster/walsummarizer.c
index ec2874c18c..c820d1f9ed 100644
--- a/src/backend/postmaster/walsummarizer.c
+++ b/src/backend/postmaster/walsummarizer.c
@@ -139,7 +139,7 @@ static XLogRecPtr redo_pointer_at_last_summary_removal = InvalidXLogRecPtr;
* GUC parameters
*/
bool summarize_wal = false;
-int wal_summary_keep_time = 10 * 24 * 60;
+int wal_summary_keep_time = 10 * HOURS_PER_DAY * MINS_PER_HOUR;
static void WalSummarizerShutdown(int code, Datum arg);
static XLogRecPtr GetLatestLSN(TimeLineID *tli);
@@ -1474,7 +1474,7 @@ MaybeRemoveOldWalSummaries(void)
* Files should only be removed if the last modification time precedes the
* cutoff time we compute here.
*/
- cutoff_time = time(NULL) - 60 * wal_summary_keep_time;
+ cutoff_time = time(NULL) - wal_summary_keep_time * SECS_PER_MINUTE;
/* Get all the summaries that currently exist. */
wslist = GetWalSummaries(0, InvalidXLogRecPtr, InvalidXLogRecPtr);
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 57d9de4dd9..1e71e7db4a 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3293,9 +3293,9 @@ struct config_int ConfigureNamesInt[] =
GUC_UNIT_MIN,
},
&wal_summary_keep_time,
- 10 * 24 * 60, /* 10 days */
+ 10 * HOURS_PER_DAY * MINS_PER_HOUR, /* 10 days */
0,
- INT_MAX,
+ INT_MAX / SECS_PER_MINUTE,
NULL, NULL, NULL
},
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-03-15 01:52 Nathan Bossart <[email protected]>
parent: Nathan Bossart <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Nathan Bossart @ 2024-03-15 01:52 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Thu, Mar 14, 2024 at 04:00:10PM -0500, Nathan Bossart wrote:
> Separately, I suppose it's probably time to revert the temporary debugging
> code adding by commit 5ddf997. I can craft a patch for that, too.
As promised...
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
Attachments:
[text/x-diff] v1-0001-Revert-Temporary-patch-to-help-debug-pg_walsummar.patch (3.0K, ../../20240315015255.GA3248600@nathanxps13/2-v1-0001-Revert-Temporary-patch-to-help-debug-pg_walsummar.patch)
download | inline diff:
From 3923e30acb1d4e21ce311b25edbcd8b96b4223d2 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <[email protected]>
Date: Thu, 14 Mar 2024 20:36:48 -0500
Subject: [PATCH v1 1/2] Revert "Temporary patch to help debug pg_walsummary
test failures."
This reverts commit 5ddf9973477729cf161b4ad0a1efd52f4fea9c88.
---
src/backend/backup/walsummary.c | 7 -------
src/bin/pg_walsummary/t/002_blocks.pl | 14 --------------
2 files changed, 21 deletions(-)
diff --git a/src/backend/backup/walsummary.c b/src/backend/backup/walsummary.c
index 4d047e1c02..322ae3c3ad 100644
--- a/src/backend/backup/walsummary.c
+++ b/src/backend/backup/walsummary.c
@@ -252,15 +252,8 @@ RemoveWalSummaryIfOlderThan(WalSummaryFile *ws, time_t cutoff_time)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not stat file \"%s\": %m", path)));
- /* XXX temporarily changed to debug buildfarm failures */
-#if 0
ereport(DEBUG2,
(errmsg_internal("removing file \"%s\"", path)));
-#else
- ereport(LOG,
- (errmsg_internal("removing file \"%s\" cutoff_time=%llu", path,
- (unsigned long long) cutoff_time)));
-#endif
}
/*
diff --git a/src/bin/pg_walsummary/t/002_blocks.pl b/src/bin/pg_walsummary/t/002_blocks.pl
index d4bae3d564..52d3bd8840 100644
--- a/src/bin/pg_walsummary/t/002_blocks.pl
+++ b/src/bin/pg_walsummary/t/002_blocks.pl
@@ -51,7 +51,6 @@ my $summarized_lsn = $node1->safe_psql('postgres', <<EOM);
SELECT MAX(end_lsn) AS summarized_lsn FROM pg_available_wal_summaries()
EOM
note("after insert, summarized through $summarized_lsn");
-note_wal_summary_dir("after insert", $node1);
# Update a row in the first block of the table and trigger a checkpoint.
$node1->safe_psql('postgres', <<EOM);
@@ -78,7 +77,6 @@ my @lines = split(/\n/, $details);
is(0+@lines, 1, "got exactly one new WAL summary");
my ($tli, $start_lsn, $end_lsn) = split(/\|/, $lines[0]);
note("examining summary for TLI $tli from $start_lsn to $end_lsn");
-note_wal_summary_dir("after new summary", $node1);
# Reconstruct the full pathname for the WAL summary file.
my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
@@ -86,7 +84,6 @@ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
split(m@/@, $start_lsn),
split(m@/@, $end_lsn);
ok(-f $filename, "WAL summary file exists");
-note_wal_summary_dir("after existence check", $node1);
# Run pg_walsummary on it. We expect exactly two blocks to be modified,
# block 0 and one other.
@@ -96,16 +93,5 @@ note($stdout);
like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified");
is($stderr, '', 'stderr is empty');
is(0+@lines, 2, "UPDATE modified 2 blocks");
-note_wal_summary_dir("after pg_walsummary run", $node1);
done_testing();
-
-# XXX. Temporary debugging code.
-sub note_wal_summary_dir
-{
- my ($flair, $node) = @_;
-
- my $wsdir = sprintf "%s/pg_wal/summaries", $node->data_dir;
- my @wsfiles = grep { $_ ne '.' && $_ ne '..' } slurp_dir($wsdir);
- note("$flair pg_wal/summaries has: @wsfiles");
-}
--
2.25.1
[text/x-diff] v1-0002-Fix-possible-overflow-in-MaybeRemoveOldWalSummari.patch (1.8K, ../../20240315015255.GA3248600@nathanxps13/3-v1-0002-Fix-possible-overflow-in-MaybeRemoveOldWalSummari.patch)
download | inline diff:
From 38dda89ee48736489d37e18dea186e90358468b0 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <[email protected]>
Date: Thu, 14 Mar 2024 20:46:02 -0500
Subject: [PATCH v1 2/2] Fix possible overflow in MaybeRemoveOldWalSummaries().
---
src/backend/postmaster/walsummarizer.c | 4 ++--
src/backend/utils/misc/guc_tables.c | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/backend/postmaster/walsummarizer.c b/src/backend/postmaster/walsummarizer.c
index ec2874c18c..c820d1f9ed 100644
--- a/src/backend/postmaster/walsummarizer.c
+++ b/src/backend/postmaster/walsummarizer.c
@@ -139,7 +139,7 @@ static XLogRecPtr redo_pointer_at_last_summary_removal = InvalidXLogRecPtr;
* GUC parameters
*/
bool summarize_wal = false;
-int wal_summary_keep_time = 10 * 24 * 60;
+int wal_summary_keep_time = 10 * HOURS_PER_DAY * MINS_PER_HOUR;
static void WalSummarizerShutdown(int code, Datum arg);
static XLogRecPtr GetLatestLSN(TimeLineID *tli);
@@ -1474,7 +1474,7 @@ MaybeRemoveOldWalSummaries(void)
* Files should only be removed if the last modification time precedes the
* cutoff time we compute here.
*/
- cutoff_time = time(NULL) - 60 * wal_summary_keep_time;
+ cutoff_time = time(NULL) - wal_summary_keep_time * SECS_PER_MINUTE;
/* Get all the summaries that currently exist. */
wslist = GetWalSummaries(0, InvalidXLogRecPtr, InvalidXLogRecPtr);
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 57d9de4dd9..1e71e7db4a 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3293,9 +3293,9 @@ struct config_int ConfigureNamesInt[] =
GUC_UNIT_MIN,
},
&wal_summary_keep_time,
- 10 * 24 * 60, /* 10 days */
+ 10 * HOURS_PER_DAY * MINS_PER_HOUR, /* 10 days */
0,
- INT_MAX,
+ INT_MAX / SECS_PER_MINUTE,
NULL, NULL, NULL
},
--
2.25.1
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-03-19 18:15 Nathan Bossart <[email protected]>
parent: Nathan Bossart <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Nathan Bossart @ 2024-03-19 18:15 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Thu, Mar 14, 2024 at 08:52:55PM -0500, Nathan Bossart wrote:
> Subject: [PATCH v1 1/2] Revert "Temporary patch to help debug pg_walsummary
> test failures."
> Subject: [PATCH v1 2/2] Fix possible overflow in MaybeRemoveOldWalSummaries().
Assuming there are no objections or feedback, I plan to commit these two
patches within the next couple of days.
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: cleanup patches for incremental backup
@ 2024-03-20 18:35 Nathan Bossart <[email protected]>
parent: Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2024-03-20 18:35 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, Mar 19, 2024 at 01:15:02PM -0500, Nathan Bossart wrote:
> Assuming there are no objections or feedback, I plan to commit these two
> patches within the next couple of days.
Committed.
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 33+ messages in thread
end of thread, other threads:[~2024-03-20 18:35 UTC | newest]
Thread overview: 33+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-10-07 03:11 [PATCH v8 4/8] Propagate changes to indisclustered to child/parents Justin Pryzby <[email protected]>
2024-01-15 16:58 Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-15 20:31 ` Re: cleanup patches for incremental backup Matthias van de Meent <[email protected]>
2024-01-16 15:39 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-16 20:22 ` Re: cleanup patches for incremental backup Matthias van de Meent <[email protected]>
2024-01-16 20:49 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-17 18:42 ` Re: cleanup patches for incremental backup Matthias van de Meent <[email protected]>
2024-01-17 20:10 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-18 09:49 ` Re: cleanup patches for incremental backup Matthias van de Meent <[email protected]>
2024-01-18 14:40 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-24 17:08 ` Re: cleanup patches for incremental backup Nathan Bossart <[email protected]>
2024-01-24 17:46 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-24 18:05 ` Re: cleanup patches for incremental backup Nathan Bossart <[email protected]>
2024-01-24 19:08 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-24 19:39 ` Re: cleanup patches for incremental backup Nathan Bossart <[email protected]>
2024-01-25 15:06 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-25 16:08 ` Re: cleanup patches for incremental backup Nathan Bossart <[email protected]>
2024-01-26 16:04 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-26 17:39 ` Re: cleanup patches for incremental backup Nathan Bossart <[email protected]>
2024-01-26 18:37 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-27 07:00 ` Re: cleanup patches for incremental backup Alexander Lakhin <[email protected]>
2024-03-14 21:00 ` Re: cleanup patches for incremental backup Nathan Bossart <[email protected]>
2024-03-15 01:52 ` Re: cleanup patches for incremental backup Nathan Bossart <[email protected]>
2024-03-19 18:15 ` Re: cleanup patches for incremental backup Nathan Bossart <[email protected]>
2024-03-20 18:35 ` Re: cleanup patches for incremental backup Nathan Bossart <[email protected]>
2024-01-27 08:00 ` Re: cleanup patches for incremental backup Alexander Lakhin <[email protected]>
2024-01-27 16:31 ` Re: cleanup patches for incremental backup Nathan Bossart <[email protected]>
2024-01-29 18:21 ` Re: cleanup patches for incremental backup Nathan Bossart <[email protected]>
2024-01-29 20:18 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-29 21:13 ` Re: cleanup patches for incremental backup Nathan Bossart <[email protected]>
2024-01-30 15:51 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-30 16:52 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
2024-01-31 15:33 ` Re: cleanup patches for incremental backup Robert Haas <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox